machine_learning_hw\exam.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "exam.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyO1XP86GX5r2ZftUJX1FN+y"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "UYTDLDB3DqQ2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 212
        },
        "outputId": "6ebd4409-15c5-4cb1-b9a6-360b866d3e51"
      },
      "source": [
        "!pip install eli5"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: eli5 in /usr/local/lib/python3.6/dist-packages (0.10.1)\n",
            "Requirement already satisfied: scikit-learn>=0.18 in /usr/local/lib/python3.6/dist-packages (from eli5) (0.22.2.post1)\n",
            "Requirement already satisfied: attrs>16.0.0 in /usr/local/lib/python3.6/dist-packages (from eli5) (19.3.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.6/dist-packages (from eli5) (2.11.2)\n",
            "Requirement already satisfied: graphviz in /usr/local/lib/python3.6/dist-packages (from eli5) (0.10.1)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from eli5) (1.4.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from eli5) (1.12.0)\n",
            "Requirement already satisfied: tabulate>=0.7.7 in /usr/local/lib/python3.6/dist-packages (from eli5) (0.8.7)\n",
            "Requirement already satisfied: numpy>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from eli5) (1.18.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.18->eli5) (0.15.1)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from jinja2->eli5) (1.1.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UxcKri3leUmN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "52a647f8-77b9-4603-c9c7-1abed2a52199"
      },
      "source": [
        "!pip install youtokentome"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: youtokentome in /usr/local/lib/python3.6/dist-packages (1.0.6)\n",
            "Requirement already satisfied: Click>=7.0 in /usr/local/lib/python3.6/dist-packages (from youtokentome) (7.1.2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bUj5MOh6RuVF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "d9a744f5-71c8-4490-eae3-7f418854a8a7"
      },
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5aZd2C5NqbCR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        },
        "outputId": "60eaf9e7-f698-40e0-f420-d3e23226c516"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.metrics import *\n",
        "from sklearn import metrics\n",
        "import gensim\n",
        "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
        "from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV\n",
        "from sklearn.decomposition import TruncatedSVD\n",
        "from sklearn.cluster import DBSCAN\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.naive_bayes import GaussianNB, MultinomialNB, BernoulliNB\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.pipeline import FeatureUnion\n",
        "from sklearn.linear_model import SGDClassifier, RidgeClassifier\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.preprocessing import FunctionTransformer\n",
        "import eli5\n",
        "from matplotlib import pyplot\n",
        "from eli5.sklearn import PermutationImportance\n",
        "from nltk.corpus import stopwords\n",
        "from string import punctuation\n",
        "from sklearn.ensemble import VotingClassifier \n",
        "from xgboost import XGBClassifier\n",
        "from sklearn.linear_model import LogisticRegression \n",
        "from sklearn.svm import SVC \n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "import tensorflow as tf\n",
        "from string import punctuation\n",
        "from collections import Counter\n",
        "import youtokentome as yttm\n",
        "import matplotlib.pyplot as plt\n",
        "import re\n",
        "%matplotlib inline"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.metrics.scorer module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.metrics. Anything that cannot be imported from sklearn.metrics is now part of the private API.\n",
            "  warnings.warn(message, FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.feature_selection.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.feature_selection. Anything that cannot be imported from sklearn.feature_selection is now part of the private API.\n",
            "  warnings.warn(message, FutureWarning)\n",
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b4_mRAqZvya9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "ca631802-58d0-4f8c-bea6-b22720a1328e"
      },
      "source": [
        "from google.colab import drive \n",
        "drive.mount('/gdrive')\n",
        "%cd /gdrive"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /gdrive; to attempt to forcibly remount, call drive.mount(\"/gdrive\", force_remount=True).\n",
            "/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dy0f5R_GxBFK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data = pd.read_csv('/gdrive/My Drive/exam/jigsaw-toxic-comment-train.csv.zip')"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fSDK50e-sSu8",
        "colab_type": "text"
      },
      "source": [
        "## 1. Описательный анализ данных с привязкой к целевой переменной (toxic) и 2 другим дополнительным колонкам (как минимум 5 статистик на каждую группу) - 1.5 балл"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hVgTHTVQrG0k",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 347
        },
        "outputId": "d6c39789-7eab-4403-8aad-ebff80c0dedf"
      },
      "source": [
        "data.head(10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0000997932d777bf</td>\n",
              "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>000103f0d9cfb60f</td>\n",
              "      <td>D'aww! He matches this background colour I'm s...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>000113f07ec002fd</td>\n",
              "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0001b41b1c6bb37e</td>\n",
              "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0001d958c54c6e35</td>\n",
              "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>00025465d4725e87</td>\n",
              "      <td>\"\\n\\nCongratulations from me as well, use the ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>0002bcb3da6cb337</td>\n",
              "      <td>COCKSUCKER BEFORE YOU PISS AROUND ON MY WORK</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>00031b1e95af7921</td>\n",
              "      <td>Your vandalism to the Matt Shirvington article...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>00037261f536c51d</td>\n",
              "      <td>Sorry if the word 'nonsense' was offensive to ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>00040093b2687caa</td>\n",
              "      <td>alignment on this subject and which are contra...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 id  ... identity_hate\n",
              "0  0000997932d777bf  ...             0\n",
              "1  000103f0d9cfb60f  ...             0\n",
              "2  000113f07ec002fd  ...             0\n",
              "3  0001b41b1c6bb37e  ...             0\n",
              "4  0001d958c54c6e35  ...             0\n",
              "5  00025465d4725e87  ...             0\n",
              "6  0002bcb3da6cb337  ...             0\n",
              "7  00031b1e95af7921  ...             0\n",
              "8  00037261f536c51d  ...             0\n",
              "9  00040093b2687caa  ...             0\n",
              "\n",
              "[10 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e4qLRnZtM_uP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "07a9a710-e5c8-4795-f35c-7219c51995b4"
      },
      "source": [
        "len(data[data['toxic']==1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "21384"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H7DaPDPWNeS7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "518d4c22-ec70-43c3-d383-a197336fa98d"
      },
      "source": [
        "len(data[data['toxic']==0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "202165"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fUub4b3eIkqK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "1598287c-e153-42d6-9346-c7ee26657451"
      },
      "source": [
        "data.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(223549, 8)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ykvhzn2CP_e2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def is_ascii(s):\n",
        "    return all(ord(c) < 128 for c in s)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f6Z4bsJ_QDsf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "stops = stopwords.words('english')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GA0ESYCPMFrI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data['text_len'] = data['comment_text'].apply(len) #длина текста\n",
        "data['text_len_tokens'] = data['comment_text'].apply(lambda x: len(x.split())) #количество токенов\n",
        "data['number_of_caps'] = data['comment_text'].apply(lambda x: len([ch for ch in x if ch.isupper()])) #количество слов с заглавной буквы\n",
        "data['number_of_nonalphanum'] = data['comment_text'].apply(lambda x: len([ch for ch in x if not ch.isalnum()])) #количество  не буквенно-цифровых символов\n",
        "data['number_of_lows'] = data['comment_text'].apply(lambda x: len([ch for ch in x if ch.islower()])) #количество строчных букв\n",
        "data['number_of_digits'] = data['comment_text'].apply(lambda x: len([ch for ch in x if ch.isdigit()])) #количество цифр\n",
        "data['number_of_ascii'] = data['comment_text'].apply(lambda x: len([ch for ch in x if is_ascii(ch)])) #количество ascii символов\n",
        "data['number_of_nonascii'] = data['comment_text'].apply(lambda x: len([ch for ch in x if not is_ascii(ch)])) #количество non-ascii символов\n",
        "data['number_of_punctuation'] = data['comment_text'].apply(lambda x: len([ch for ch in x if ch in punctuation])) #количество знаков пунктуации\n",
        "data['number_of_stops'] = data['comment_text'].apply(lambda x: len([ch for ch in x if ch in stops])) #количество стоп-слов"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6gQrAt86SL6G",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        },
        "outputId": "c2a3bfb8-897d-4a50-bda5-439ba52731b8"
      },
      "source": [
        "data.head(3)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "      <th>text_len</th>\n",
              "      <th>text_len_tokens</th>\n",
              "      <th>number_of_caps</th>\n",
              "      <th>number_of_nonalphanum</th>\n",
              "      <th>number_of_lows</th>\n",
              "      <th>number_of_digits</th>\n",
              "      <th>number_of_ascii</th>\n",
              "      <th>number_of_nonascii</th>\n",
              "      <th>number_of_punctuation</th>\n",
              "      <th>number_of_stops</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0000997932d777bf</td>\n",
              "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>264</td>\n",
              "      <td>43</td>\n",
              "      <td>17</td>\n",
              "      <td>52</td>\n",
              "      <td>186</td>\n",
              "      <td>9</td>\n",
              "      <td>264</td>\n",
              "      <td>0</td>\n",
              "      <td>10</td>\n",
              "      <td>84</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>000103f0d9cfb60f</td>\n",
              "      <td>D'aww! He matches this background colour I'm s...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>112</td>\n",
              "      <td>17</td>\n",
              "      <td>8</td>\n",
              "      <td>29</td>\n",
              "      <td>65</td>\n",
              "      <td>10</td>\n",
              "      <td>112</td>\n",
              "      <td>0</td>\n",
              "      <td>12</td>\n",
              "      <td>29</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>000113f07ec002fd</td>\n",
              "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>233</td>\n",
              "      <td>42</td>\n",
              "      <td>4</td>\n",
              "      <td>47</td>\n",
              "      <td>182</td>\n",
              "      <td>0</td>\n",
              "      <td>233</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>99</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 id  ... number_of_stops\n",
              "0  0000997932d777bf  ...              84\n",
              "1  000103f0d9cfb60f  ...              29\n",
              "2  000113f07ec002fd  ...              99\n",
              "\n",
              "[3 rows x 18 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "is6Qp1Vy2-4C",
        "colab_type": "text"
      },
      "source": [
        "### Отдельные колонки"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U0AxvX6XXbeQ",
        "colab_type": "text"
      },
      "source": [
        "### Toxic/Non-toxic"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6pZtJvZgWNGZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 227
        },
        "outputId": "a3ea7fe3-d49c-418e-8f53-a28e4990e521"
      },
      "source": [
        "data.groupby('toxic')['text_len', 'text_len_tokens', 'number_of_caps', 'number_of_nonalphanum', 'number_of_lows', 'number_of_digits', \n",
        "                      'number_of_ascii', 'number_of_nonascii', 'number_of_punctuation', 'number_of_stops'].agg(['mean'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>text_len</th>\n",
              "      <th>text_len_tokens</th>\n",
              "      <th>number_of_caps</th>\n",
              "      <th>number_of_nonalphanum</th>\n",
              "      <th>number_of_lows</th>\n",
              "      <th>number_of_digits</th>\n",
              "      <th>number_of_ascii</th>\n",
              "      <th>number_of_nonascii</th>\n",
              "      <th>number_of_punctuation</th>\n",
              "      <th>number_of_stops</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>toxic</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>402.691178</td>\n",
              "      <td>68.415161</td>\n",
              "      <td>14.484189</td>\n",
              "      <td>89.072456</td>\n",
              "      <td>294.922009</td>\n",
              "      <td>3.248698</td>\n",
              "      <td>400.972142</td>\n",
              "      <td>1.719036</td>\n",
              "      <td>16.974704</td>\n",
              "      <td>141.799609</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>280.604097</td>\n",
              "      <td>48.573466</td>\n",
              "      <td>44.689675</td>\n",
              "      <td>65.266741</td>\n",
              "      <td>168.618921</td>\n",
              "      <td>1.995885</td>\n",
              "      <td>280.409512</td>\n",
              "      <td>0.194585</td>\n",
              "      <td>14.257202</td>\n",
              "      <td>80.432941</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         text_len text_len_tokens  ... number_of_punctuation number_of_stops\n",
              "             mean            mean  ...                  mean            mean\n",
              "toxic                              ...                                      \n",
              "0      402.691178       68.415161  ...             16.974704      141.799609\n",
              "1      280.604097       48.573466  ...             14.257202       80.432941\n",
              "\n",
              "[2 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4pt0HKP7XTWe",
        "colab_type": "text"
      },
      "source": [
        "1. Средняя длина нетоксичного комментария выше, чем у токсичного. \n",
        "2. Среднее количество слов у нетоксичных текстов также больше. \n",
        "3. Среднее количество используемых заглавных букв у токсичных сообщений в несколько рах выше, чем у нетоксичных.\n",
        "4. Среднее количество  не буквенно-цифровых символов в нетоксичных комментариях выше.\n",
        "5. Количество строчных букв в токсичных комментариях меньше, чем в нетоксичных.\n",
        "6. Цифр в нетоксичных комментариях также в среднем больше. \n",
        "7. Ascii-символов в нетоксичных комментариях больше.\n",
        "8. Не-ascii символов в целом по комментариям практически нет, но если они встречаются, то чаще в нетоксичных комментариях.\n",
        "9. Оба типа комментариев имеют практически одинаковое среднее количество знаков препинания. В нетоксичных комментариях их чуть больше. \n",
        "10. Используемых стоп-слов в нетоксичных комментариях гораздо больше, чем в токсичных."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "992m3sndR2Yq",
        "colab_type": "text"
      },
      "source": [
        "### Severe_toxic/Non-severe_toxic"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kwwev4HSR0VL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 227
        },
        "outputId": "aac006eb-c550-41fe-9fbb-ba029b4dc9e8"
      },
      "source": [
        "data.groupby('severe_toxic')['text_len', 'text_len_tokens', 'number_of_caps', 'number_of_nonalphanum', 'number_of_lows', 'number_of_digits', \n",
        "                      'number_of_ascii', 'number_of_nonascii', 'number_of_punctuation', 'number_of_stops'].agg(['mean'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>text_len</th>\n",
              "      <th>text_len_tokens</th>\n",
              "      <th>number_of_caps</th>\n",
              "      <th>number_of_nonalphanum</th>\n",
              "      <th>number_of_lows</th>\n",
              "      <th>number_of_digits</th>\n",
              "      <th>number_of_ascii</th>\n",
              "      <th>number_of_nonascii</th>\n",
              "      <th>number_of_punctuation</th>\n",
              "      <th>number_of_stops</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>severe_toxic</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>389.999386</td>\n",
              "      <td>66.358270</td>\n",
              "      <td>15.988131</td>\n",
              "      <td>86.500630</td>\n",
              "      <td>283.491852</td>\n",
              "      <td>3.136258</td>\n",
              "      <td>388.413228</td>\n",
              "      <td>1.586158</td>\n",
              "      <td>16.591014</td>\n",
              "      <td>136.307514</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>505.456677</td>\n",
              "      <td>84.462793</td>\n",
              "      <td>173.841998</td>\n",
              "      <td>120.072375</td>\n",
              "      <td>209.248726</td>\n",
              "      <td>2.293068</td>\n",
              "      <td>505.345566</td>\n",
              "      <td>0.111111</td>\n",
              "      <td>30.690112</td>\n",
              "      <td>93.232926</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                text_len text_len_tokens  ... number_of_punctuation number_of_stops\n",
              "                    mean            mean  ...                  mean            mean\n",
              "severe_toxic                              ...                                      \n",
              "0             389.999386       66.358270  ...             16.591014      136.307514\n",
              "1             505.456677       84.462793  ...             30.690112       93.232926\n",
              "\n",
              "[2 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AyI3mFJaSK1w",
        "colab_type": "text"
      },
      "source": [
        "1. Средняя длина сильно токсичных комментариев гораздо больше. \n",
        "2. Среднее количество слов в сильно токсичных комментариях выше.\n",
        "3. Количество заглавных букв в сильно токсичных текстах во много раз превышает количество в обычных комментариях.\n",
        "4. Среднее количество не буквенно-цифровых символов в токсичных комментариях выше.\n",
        "5. Строчных букв в обычных комментариях больше. \n",
        "6. В обоих типах комментариев цифр мало, но в обычных комментариях в среднем чуть больше.\n",
        "7. Ascii-символов больше в сильно токсичных комментариях.\n",
        "8. Не-ascii символы практически отсутствуют в обоих типах. В обычных чуть больше.\n",
        "9. Количество знаков пунктуации в среднем почти в 2 раза больше в токсичных комментариях.\n",
        "10. Количество стоп-слов в обычных комментариях в среднем больше. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V8zOtYEPSMIy",
        "colab_type": "text"
      },
      "source": [
        "### Obscene/Non-obscene"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lWFqZkxUSMRv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 227
        },
        "outputId": "2a4d663e-43cd-4f15-a674-2c1192936d93"
      },
      "source": [
        "data.groupby('obscene')['text_len', 'text_len_tokens', 'number_of_caps', 'number_of_nonalphanum', 'number_of_lows', 'number_of_digits', \n",
        "                      'number_of_ascii', 'number_of_nonascii', 'number_of_punctuation', 'number_of_stops'].agg(['mean'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>text_len</th>\n",
              "      <th>text_len_tokens</th>\n",
              "      <th>number_of_caps</th>\n",
              "      <th>number_of_nonalphanum</th>\n",
              "      <th>number_of_lows</th>\n",
              "      <th>number_of_digits</th>\n",
              "      <th>number_of_ascii</th>\n",
              "      <th>number_of_nonascii</th>\n",
              "      <th>number_of_punctuation</th>\n",
              "      <th>number_of_stops</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>obscene</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>397.332876</td>\n",
              "      <td>67.559707</td>\n",
              "      <td>15.299590</td>\n",
              "      <td>88.032052</td>\n",
              "      <td>289.883993</td>\n",
              "      <td>3.193029</td>\n",
              "      <td>395.678353</td>\n",
              "      <td>1.654523</td>\n",
              "      <td>16.850087</td>\n",
              "      <td>139.417163</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>280.951730</td>\n",
              "      <td>48.362109</td>\n",
              "      <td>53.490033</td>\n",
              "      <td>65.257743</td>\n",
              "      <td>160.178748</td>\n",
              "      <td>2.011367</td>\n",
              "      <td>280.794481</td>\n",
              "      <td>0.157249</td>\n",
              "      <td>14.358072</td>\n",
              "      <td>75.193822</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           text_len text_len_tokens  ... number_of_punctuation number_of_stops\n",
              "               mean            mean  ...                  mean            mean\n",
              "obscene                              ...                                      \n",
              "0        397.332876       67.559707  ...             16.850087      139.417163\n",
              "1        280.951730       48.362109  ...             14.358072       75.193822\n",
              "\n",
              "[2 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qID8tEdrSMh7",
        "colab_type": "text"
      },
      "source": [
        "1. Средняя длина грубых комментарией намного меньше, чем обычных.\n",
        "2. Количество слов в грубых текстах также меньше в среднем на 19.\n",
        "3. Заглавных букв в грубых комментариях во много раз больше.\n",
        "4. Среднее количество не буквенно-цифровых символов в грубых комментариях меньше.\n",
        "5. Негрубые тексты содержат гораздо больше строчных букв. \n",
        "6. Оба типа комментариев содержат мало цифр. В негрубых их в среднем на 1 больше.\n",
        "7. Количество ascii-символов практически полностью совпадает со средней длиной комментариев в обих типах.\n",
        "8. В негрубых в среднем встречается 1.6 не ascii-символов, что чуть больше, чем в грубых.\n",
        "9. Количество знаков пунктуации практически одинаковое, но в негрубых их на 2 больше, чем в грубых.\n",
        "10. В грубых комментариях используется меньше стоп-слов, чем в негрубых почти в 2 раза."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nnJq1D5oSbqI",
        "colab_type": "text"
      },
      "source": [
        "### Threat/Non-threat"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "70T97KdAScIz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 227
        },
        "outputId": "e2d30373-80e1-4386-a9dd-af1a5653690d"
      },
      "source": [
        "data.groupby('threat')['text_len', 'text_len_tokens', 'number_of_caps', 'number_of_nonalphanum', 'number_of_lows', 'number_of_digits', \n",
        "                      'number_of_ascii', 'number_of_nonascii', 'number_of_punctuation', 'number_of_stops'].agg(['mean'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>text_len</th>\n",
              "      <th>text_len_tokens</th>\n",
              "      <th>number_of_caps</th>\n",
              "      <th>number_of_nonalphanum</th>\n",
              "      <th>number_of_lows</th>\n",
              "      <th>number_of_digits</th>\n",
              "      <th>number_of_ascii</th>\n",
              "      <th>number_of_nonascii</th>\n",
              "      <th>number_of_punctuation</th>\n",
              "      <th>number_of_stops</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>threat</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>391.241807</td>\n",
              "      <td>66.547128</td>\n",
              "      <td>17.149430</td>\n",
              "      <td>86.796312</td>\n",
              "      <td>283.283766</td>\n",
              "      <td>3.134856</td>\n",
              "      <td>389.665157</td>\n",
              "      <td>1.576649</td>\n",
              "      <td>16.686637</td>\n",
              "      <td>136.141282</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>316.910015</td>\n",
              "      <td>56.825835</td>\n",
              "      <td>89.866473</td>\n",
              "      <td>86.460087</td>\n",
              "      <td>139.383164</td>\n",
              "      <td>1.188679</td>\n",
              "      <td>316.448476</td>\n",
              "      <td>0.461538</td>\n",
              "      <td>25.809869</td>\n",
              "      <td>67.416546</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          text_len text_len_tokens  ... number_of_punctuation number_of_stops\n",
              "              mean            mean  ...                  mean            mean\n",
              "threat                              ...                                      \n",
              "0       391.241807       66.547128  ...             16.686637      136.141282\n",
              "1       316.910015       56.825835  ...             25.809869       67.416546\n",
              "\n",
              "[2 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fs09jba7Scpb",
        "colab_type": "text"
      },
      "source": [
        "1. Длина текстов с угрозой меньше, чем текстов без угрозы.\n",
        "2. Количество слов в текстах без угрозы на 10 больше. \n",
        "3. Количество заглавных букв в текстах с угрозой во много раз больше, чем в текстах без.\n",
        "4. Среднее количество не буквенно-цифровых символов одинаково.\n",
        "5. Количество строчных букв в текстах без угрозы больше, чем в 2 раза.\n",
        "6. Количество цифр в обоих типах мало, но в текстах без угрозы их в среднем чуть больше.\n",
        "7. Количество ascii-символов практически полностью совпадает со средней длиной комментариев в обих типах.\n",
        "8. Не ascii-символы практически отсутствуют в обоих типах. В текстах без угрозы в среднем их 1.6.\n",
        "9. Знаки пунктуации присутствуют в обоих типах комментариев и их количество почти одинаковое.\n",
        "10. Количество использованных стоп-слов больше в текстах без угрозы в 2 раза. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bCL5bm8hSrBd",
        "colab_type": "text"
      },
      "source": [
        "### Insult/Non-insult"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6IHhN3H8SrMp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 227
        },
        "outputId": "e8f232c2-2c2b-47fe-ef12-159ab58e8495"
      },
      "source": [
        "data.groupby('insult')['text_len', 'text_len_tokens', 'number_of_caps', 'number_of_nonalphanum', 'number_of_lows', 'number_of_digits', \n",
        "                      'number_of_ascii', 'number_of_nonascii', 'number_of_punctuation', 'number_of_stops'].agg(['mean'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>text_len</th>\n",
              "      <th>text_len_tokens</th>\n",
              "      <th>number_of_caps</th>\n",
              "      <th>number_of_nonalphanum</th>\n",
              "      <th>number_of_lows</th>\n",
              "      <th>number_of_digits</th>\n",
              "      <th>number_of_ascii</th>\n",
              "      <th>number_of_nonascii</th>\n",
              "      <th>number_of_punctuation</th>\n",
              "      <th>number_of_stops</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>insult</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>397.340234</td>\n",
              "      <td>67.544536</td>\n",
              "      <td>15.573931</td>\n",
              "      <td>88.053302</td>\n",
              "      <td>289.597522</td>\n",
              "      <td>3.194615</td>\n",
              "      <td>395.690669</td>\n",
              "      <td>1.649565</td>\n",
              "      <td>16.879710</td>\n",
              "      <td>139.233245</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>272.206476</td>\n",
              "      <td>47.227176</td>\n",
              "      <td>51.163393</td>\n",
              "      <td>63.174452</td>\n",
              "      <td>155.965057</td>\n",
              "      <td>1.894197</td>\n",
              "      <td>272.066879</td>\n",
              "      <td>0.139597</td>\n",
              "      <td>13.617569</td>\n",
              "      <td>73.897381</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          text_len text_len_tokens  ... number_of_punctuation number_of_stops\n",
              "              mean            mean  ...                  mean            mean\n",
              "insult                              ...                                      \n",
              "0       397.340234       67.544536  ...             16.879710      139.233245\n",
              "1       272.206476       47.227176  ...             13.617569       73.897381\n",
              "\n",
              "[2 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K_PX3EINSrb0",
        "colab_type": "text"
      },
      "source": [
        "1. Средняя длина комментария с оскорблением меньше, чем обычный комментарий.\n",
        "2. В обычном комментарии в среднем на 20 слов больше, чем в тексте с оскорблением.\n",
        "3. Количество заглавных букв в тексте с оскорблением в несколько раз больше.\n",
        "4. Среднее количество не буквенно-цифровых символов в обычных комментариях больше, чем в оскорбительных.\n",
        "5. Количество строчных букв почти в 2 раза больше в комментариях без оскорбления.\n",
        "6. Количество цифр в обоих типах мало, но в текстах без оскорбления их в среднем чуть больше.\n",
        "7. Количество ascii-символов практически полностью совпадает со средней длиной комментариев в обих типах.\n",
        "8. Не ascii-символы практически отсутствуют в обоих типах. В текстах без оскорбления в среднем их 1.6.\n",
        "9. Знаки пунктуации присутствуют в обоих типах комментариев и их количество почти одинаковое.\n",
        "10. Количество использованных стоп-слов больше в текстах без оскорбления в 2 раза."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tThoJsZaTAku",
        "colab_type": "text"
      },
      "source": [
        "### Identity_hate/Non-identity_hate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WV6Yml-rTA1D",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 227
        },
        "outputId": "8e6708f7-7b8d-4d18-92b1-5a0228adfc25"
      },
      "source": [
        "data.groupby('identity_hate')['text_len', 'text_len_tokens', 'number_of_caps', 'number_of_nonalphanum', 'number_of_lows', 'number_of_digits', \n",
        "                      'number_of_ascii', 'number_of_nonascii', 'number_of_punctuation', 'number_of_stops'].agg(['mean'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>text_len</th>\n",
              "      <th>text_len_tokens</th>\n",
              "      <th>number_of_caps</th>\n",
              "      <th>number_of_nonalphanum</th>\n",
              "      <th>number_of_lows</th>\n",
              "      <th>number_of_digits</th>\n",
              "      <th>number_of_ascii</th>\n",
              "      <th>number_of_nonascii</th>\n",
              "      <th>number_of_punctuation</th>\n",
              "      <th>number_of_stops</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>identity_hate</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>391.744793</td>\n",
              "      <td>66.651428</td>\n",
              "      <td>16.774599</td>\n",
              "      <td>86.960259</td>\n",
              "      <td>283.989184</td>\n",
              "      <td>3.137812</td>\n",
              "      <td>390.158234</td>\n",
              "      <td>1.586559</td>\n",
              "      <td>16.740187</td>\n",
              "      <td>136.492779</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>314.438829</td>\n",
              "      <td>52.473784</td>\n",
              "      <td>80.022201</td>\n",
              "      <td>69.538498</td>\n",
              "      <td>162.665092</td>\n",
              "      <td>2.192253</td>\n",
              "      <td>314.261691</td>\n",
              "      <td>0.177137</td>\n",
              "      <td>14.054795</td>\n",
              "      <td>77.008503</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 text_len text_len_tokens  ... number_of_punctuation number_of_stops\n",
              "                     mean            mean  ...                  mean            mean\n",
              "identity_hate                              ...                                      \n",
              "0              391.744793       66.651428  ...             16.740187      136.492779\n",
              "1              314.438829       52.473784  ...             14.054795       77.008503\n",
              "\n",
              "[2 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B141c7MdTBCD",
        "colab_type": "text"
      },
      "source": [
        "1. Средняя длина комментария без ненависти личности больше на примерно 75 слов.\n",
        "2. Обычные комментарии в среднем имеют больше слов.\n",
        "3. Количество заглавных букв в тексте с ненавистью личности в несколько раз больше.\n",
        "4. Среднее количество не буквенно-цифровых символов в обычных комментариях больше. \n",
        "5. Количество строчных букв почти в 2 раза больше в комментариях без ненависти.\n",
        "6. Количество цифр в обоих типах мало, но в текстах без оскорбления их в среднем чуть больше.\n",
        "7. Количество ascii-символов практически полностью совпадает со средней длиной комментариев в обих типах.\n",
        "8. Не ascii-символы практически отсутствуют в обоих типах. В текстах без ненависти в среднем их 1.58.\n",
        "9. Знаки пунктуации присутствуют в обоих типах комментариев и их количество почти одинаковое.\n",
        "10. Количество использованных стоп-слов больше в текстах без ненависти меньше, чем в 2 раза."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GOkEwsDqWqMO",
        "colab_type": "text"
      },
      "source": [
        "### Привязка к другим колонкам"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P8Eb18_63Zxj",
        "colab_type": "text"
      },
      "source": [
        "### Toxic/Insult"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qS5n-UcsW1Ol",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 263
        },
        "outputId": "58584c9d-8a44-4704-a83a-b929746cdc3c"
      },
      "source": [
        "data.groupby(['toxic', 'insult'])['text_len', 'text_len_tokens', 'number_of_caps', 'number_of_nonalphanum', 'number_of_lows', 'number_of_digits', \n",
        "                      'number_of_ascii', 'number_of_nonascii', 'number_of_punctuation', 'number_of_stops'].agg(['mean'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>text_len</th>\n",
              "      <th>text_len_tokens</th>\n",
              "      <th>number_of_caps</th>\n",
              "      <th>number_of_nonalphanum</th>\n",
              "      <th>number_of_lows</th>\n",
              "      <th>number_of_digits</th>\n",
              "      <th>number_of_ascii</th>\n",
              "      <th>number_of_nonascii</th>\n",
              "      <th>number_of_punctuation</th>\n",
              "      <th>number_of_stops</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>toxic</th>\n",
              "      <th>insult</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
              "      <th>0</th>\n",
              "      <td>402.813001</td>\n",
              "      <td>68.433497</td>\n",
              "      <td>14.441753</td>\n",
              "      <td>89.097684</td>\n",
              "      <td>295.067930</td>\n",
              "      <td>3.239016</td>\n",
              "      <td>401.089875</td>\n",
              "      <td>1.723127</td>\n",
              "      <td>16.978293</td>\n",
              "      <td>141.864081</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>362.961165</td>\n",
              "      <td>62.435275</td>\n",
              "      <td>28.323625</td>\n",
              "      <td>80.844660</td>\n",
              "      <td>247.333333</td>\n",
              "      <td>6.406149</td>\n",
              "      <td>362.576052</td>\n",
              "      <td>0.385113</td>\n",
              "      <td>15.804207</td>\n",
              "      <td>120.773463</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
              "      <th>0</th>\n",
              "      <td>294.234997</td>\n",
              "      <td>50.796784</td>\n",
              "      <td>36.903814</td>\n",
              "      <td>68.377454</td>\n",
              "      <td>186.536736</td>\n",
              "      <td>2.358104</td>\n",
              "      <td>293.971303</td>\n",
              "      <td>0.263694</td>\n",
              "      <td>15.022434</td>\n",
              "      <td>89.669097</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>266.957889</td>\n",
              "      <td>46.347651</td>\n",
              "      <td>52.484278</td>\n",
              "      <td>62.152536</td>\n",
              "      <td>150.680984</td>\n",
              "      <td>1.633258</td>\n",
              "      <td>266.832491</td>\n",
              "      <td>0.125398</td>\n",
              "      <td>13.491110</td>\n",
              "      <td>71.186412</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                text_len text_len_tokens  ... number_of_punctuation number_of_stops\n",
              "                    mean            mean  ...                  mean            mean\n",
              "toxic insult                              ...                                      \n",
              "0     0       402.813001       68.433497  ...             16.978293      141.864081\n",
              "      1       362.961165       62.435275  ...             15.804207      120.773463\n",
              "1     0       294.234997       50.796784  ...             15.022434       89.669097\n",
              "      1       266.957889       46.347651  ...             13.491110       71.186412\n",
              "\n",
              "[4 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9nMJL4OP3doQ",
        "colab_type": "text"
      },
      "source": [
        "1. Non-toxic/non-insult имеют самую большую среднюю длину. В non-toxic/insult комментариях длина текста больше, чем просто в toxic/non-insult. Toxic/insult в свою очередь имеют самые короткие тексты.\n",
        "2. Non-toxic/non-insult имеют самое большое количество слов. В insult комментариях слов в среднем на 12 больше, чем просто в toxic/non-insult. Toxic/insult в свою очередь имеют меньше всего слов.\n",
        "3. Заглавных букв больше всего в комментариях имеющие обе метки toxic/insult. Меньше всего в обычных комментариях без меток. Токсичные комментарии имеют больше заглавных букв, чем оскорбительные.\n",
        "4. Среднее количество не буквенно-цифровых символов в обычных комментариях больше. Toxic/non-insult комментарии имеют меньше данных символов, чем non-toxic/insult. \n",
        "5. Количество строчных букв больше всего в комментариях без меток. Меньше всего строчных букв в комментариях с обеими метками toxic и insult. \n",
        "6. В комментариях всех типов используется мало цифр. Больше всего в текстах non-toxic/insult, меньше всего в toxic/insult. \n",
        "7. Количество ascii-символов практически полностью совпадает со средней длиной комментариев во всех типах.\n",
        "8. Не ascii-символы практически отсутствуют в данных типах.\n",
        "9. Знаков пунктуации больше всего в обычных комментариях. В комментариях toxic/insult их наименьшее количество.\n",
        "10. Больше всего стоп-слов в обычных комментариях. Меньше всего в toxic/insult. В toxic/non-insult их меньше, чем в non-toxic/insult."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DdSSDJek3fZK",
        "colab_type": "text"
      },
      "source": [
        "### Toxic/Obscene"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OIN5M3Pp3ljX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 263
        },
        "outputId": "179c44e6-fb0d-4007-a44f-93443323abb9"
      },
      "source": [
        "data.groupby(['toxic', 'obscene'])['text_len', 'text_len_tokens', 'number_of_caps', 'number_of_nonalphanum', 'number_of_lows', 'number_of_digits', \n",
        "                      'number_of_ascii', 'number_of_nonascii', 'number_of_punctuation', 'number_of_stops'].agg(['mean'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>text_len</th>\n",
              "      <th>text_len_tokens</th>\n",
              "      <th>number_of_caps</th>\n",
              "      <th>number_of_nonalphanum</th>\n",
              "      <th>number_of_lows</th>\n",
              "      <th>number_of_digits</th>\n",
              "      <th>number_of_ascii</th>\n",
              "      <th>number_of_nonascii</th>\n",
              "      <th>number_of_punctuation</th>\n",
              "      <th>number_of_stops</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
              "      <th>0</th>\n",
              "      <td>402.480948</td>\n",
              "      <td>68.373545</td>\n",
              "      <td>14.440551</td>\n",
              "      <td>89.018316</td>\n",
              "      <td>294.816601</td>\n",
              "      <td>3.239035</td>\n",
              "      <td>400.758162</td>\n",
              "      <td>1.722786</td>\n",
              "      <td>16.961355</td>\n",
              "      <td>141.744579</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>474.761905</td>\n",
              "      <td>82.681973</td>\n",
              "      <td>29.443878</td>\n",
              "      <td>107.632653</td>\n",
              "      <td>331.057823</td>\n",
              "      <td>6.561224</td>\n",
              "      <td>474.328231</td>\n",
              "      <td>0.433673</td>\n",
              "      <td>21.551020</td>\n",
              "      <td>160.664966</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
              "      <th>0</th>\n",
              "      <td>291.786412</td>\n",
              "      <td>50.874288</td>\n",
              "      <td>32.911717</td>\n",
              "      <td>67.811534</td>\n",
              "      <td>188.754984</td>\n",
              "      <td>2.249797</td>\n",
              "      <td>291.531428</td>\n",
              "      <td>0.254984</td>\n",
              "      <td>14.568857</td>\n",
              "      <td>91.700163</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>271.086738</td>\n",
              "      <td>46.615218</td>\n",
              "      <td>54.713989</td>\n",
              "      <td>63.100848</td>\n",
              "      <td>151.480956</td>\n",
              "      <td>1.779778</td>\n",
              "      <td>270.943560</td>\n",
              "      <td>0.143179</td>\n",
              "      <td>13.991949</td>\n",
              "      <td>70.843317</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 text_len text_len_tokens  ... number_of_punctuation number_of_stops\n",
              "                     mean            mean  ...                  mean            mean\n",
              "toxic obscene                              ...                                      \n",
              "0     0        402.480948       68.373545  ...             16.961355      141.744579\n",
              "      1        474.761905       82.681973  ...             21.551020      160.664966\n",
              "1     0        291.786412       50.874288  ...             14.568857       91.700163\n",
              "      1        271.086738       46.615218  ...             13.991949       70.843317\n",
              "\n",
              "[4 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MTmOT5MU3pVA",
        "colab_type": "text"
      },
      "source": [
        "1. Non-toxic/obscene комментарии являются самыми длинными. Toxic/obscene имеют самую маленькую длину.\n",
        "2. Больше всего слов в грубых комментариях non-toxic/obscene. Меньше всего в toxic/obscene вместе.\n",
        "3. Наибольшее количество заглавных букв наблюдается в toxic/obscene комментариях. В нормальных комментариях их меньше всего. Грубые и токсичные комментарии отдельно имеют примерно одинаковое количество заглавных букв.\n",
        "4. Среднее количество не буквенно-цифровых символов в грубых комментариях больше. Toxic/obscene комментарии имеют меньше всего данных символов.\n",
        "5. Больше всего строчных букв в грубых комментариях non-toxic/obscene. Toxic/obscene имеют меньше всего данных букв.\n",
        "6. В комментариях всех типов используется мало цифр. Больше всего в текстах obscene, меньше всего в toxic/obscene.\n",
        "7. Количество ascii-символов практически полностью совпадает со средней длиной комментариев в обих типах.\n",
        "8. Не ascii-символы практически отсутствуют в данных типах.\n",
        "9. Больше всего знаков пунктуации в грубых комментариях non-toxic/obscene.\n",
        "10. Non-toxic/obscene грубые комментарии имеют больше всего стоп-слов. Меньше всего в toxic/obscene."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h0WOh5Ky3rHv",
        "colab_type": "text"
      },
      "source": [
        "### Toxic/Threat"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9yADbYMS3qfI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 263
        },
        "outputId": "341482fb-c778-4a63-901d-88c94ff48ea6"
      },
      "source": [
        "data.groupby(['toxic', 'threat'])['text_len', 'text_len_tokens', 'number_of_caps', 'number_of_nonalphanum', 'number_of_lows', 'number_of_digits', \n",
        "                      'number_of_ascii', 'number_of_nonascii', 'number_of_punctuation', 'number_of_stops'].agg(['mean'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>text_len</th>\n",
              "      <th>text_len_tokens</th>\n",
              "      <th>number_of_caps</th>\n",
              "      <th>number_of_nonalphanum</th>\n",
              "      <th>number_of_lows</th>\n",
              "      <th>number_of_digits</th>\n",
              "      <th>number_of_ascii</th>\n",
              "      <th>number_of_nonascii</th>\n",
              "      <th>number_of_punctuation</th>\n",
              "      <th>number_of_stops</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "      <th>mean</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>toxic</th>\n",
              "      <th>threat</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
              "      <th>0</th>\n",
              "      <td>402.693648</td>\n",
              "      <td>68.415233</td>\n",
              "      <td>14.473087</td>\n",
              "      <td>89.073487</td>\n",
              "      <td>294.934681</td>\n",
              "      <td>3.248400</td>\n",
              "      <td>400.974492</td>\n",
              "      <td>1.719156</td>\n",
              "      <td>16.975268</td>\n",
              "      <td>141.805764</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>388.428571</td>\n",
              "      <td>68.000000</td>\n",
              "      <td>78.600000</td>\n",
              "      <td>83.114286</td>\n",
              "      <td>221.742857</td>\n",
              "      <td>4.971429</td>\n",
              "      <td>387.400000</td>\n",
              "      <td>1.028571</td>\n",
              "      <td>13.714286</td>\n",
              "      <td>106.257143</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
              "      <th>0</th>\n",
              "      <td>279.579450</td>\n",
              "      <td>48.331983</td>\n",
              "      <td>43.245393</td>\n",
              "      <td>64.592475</td>\n",
              "      <td>169.680318</td>\n",
              "      <td>2.027738</td>\n",
              "      <td>279.392330</td>\n",
              "      <td>0.187120</td>\n",
              "      <td>13.872311</td>\n",
              "      <td>80.909165</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>313.082569</td>\n",
              "      <td>56.227829</td>\n",
              "      <td>90.469419</td>\n",
              "      <td>86.639144</td>\n",
              "      <td>134.975535</td>\n",
              "      <td>0.986239</td>\n",
              "      <td>312.651376</td>\n",
              "      <td>0.431193</td>\n",
              "      <td>26.457187</td>\n",
              "      <td>65.337920</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                text_len text_len_tokens  ... number_of_punctuation number_of_stops\n",
              "                    mean            mean  ...                  mean            mean\n",
              "toxic threat                              ...                                      \n",
              "0     0       402.693648       68.415233  ...             16.975268      141.805764\n",
              "      1       388.428571       68.000000  ...             13.714286      106.257143\n",
              "1     0       279.579450       48.331983  ...             13.872311       80.909165\n",
              "      1       313.082569       56.227829  ...             26.457187       65.337920\n",
              "\n",
              "[4 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xYL5Ivg33yzf",
        "colab_type": "text"
      },
      "source": [
        "1. Обычные комментарии самые длинные. Самыми короткими являются комментарии типа toxic/non-threat. В non-toxic/threat также наблюдается большая средняя длина. \n",
        "2. Обычные комментарии и non-toxic/threat имеют одинаковое среднее количество слов. Меньше всего слов в toxic/non-threat комментариях. \n",
        "3. Заглавных букв больше всего в toxic/threat комментариях. Меньше всего в обычных. Тексты с угрозой имеют больше заглавных букв, чем токсичные.\n",
        "4. Среднее количество не буквенно-цифровых символов незначительно отличается. В toxic/non-threat комментариях их меньше всего. В остальных типах их примерно одинаково.\n",
        "5. Больше всего строчных букв в обычных комментариях. Меньше всего в toxic/threat. Non-toxic/threat имеют больше строчных букв, чем toxic/non threat. \n",
        "6. В комментариях всех типов используется мало цифр. Больше всего в текстах threat, меньше всего в toxic/threat.\n",
        "7. Количество ascii-символов практически полностью совпадает со средней длиной комментариев в обих типах.\n",
        "8. Не ascii-символы практически отсутствуют в данных типах.\n",
        "9. Больше всего знаков пунктуации в текстах toxic/threat. В toxic/non-threat и non-toxic/threat их примерно одинаково.\n",
        "10. Наибольшее количество стоп-слов наблюдается в текстах комментариев типа non-toxic/non-threat. Меньше всего в toxic/threat. В комментариях non-toxic/threat их больше, чем в toxic/non-threat. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R9TVfOYR0Xs5",
        "colab_type": "text"
      },
      "source": [
        "## 2. Бейзлайн модель из sklearn (векторайзер + модель) с отбором признаков (через l1 регуляризацию, на глаз через анализ важных параметров или через permutation importance) - 2 балл"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gV4nfdYA96F7",
        "colab_type": "text"
      },
      "source": [
        "### Бейзлайн на всех данных"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d2gwR4gt3zx7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = data['comment_text']\n",
        "y = data['toxic']"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IPvLM4wmyJo6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vMb6eTyd4vyh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "count_vect = CountVectorizer()"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aCHukUg84zql",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "3bd6b818-bbcd-40bc-e495-ae8c8b5ea55f"
      },
      "source": [
        "X_train = count_vect.fit_transform(X_train)\n",
        "X_train.shape"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(149777, 207901)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yajKEe705QDb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7f41239b-56a8-4d37-b8ea-9752af3a87c5"
      },
      "source": [
        "X_test = count_vect.transform(X_test)\n",
        "X_test.shape"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(73772, 207901)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mmsx7fQf5tKZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "clf = SGDClassifier().fit(X_train, y_train)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YUMX-Hv26p97",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_pred = clf.predict(X_test)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hTz_FEH09KR4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 176
        },
        "outputId": "29663b1c-d0c3-4bb8-cfff-e1617bf752ad"
      },
      "source": [
        "print(classification_report(y_test, y_pred))"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.99      0.97     66765\n",
            "           1       0.85      0.55      0.67      7007\n",
            "\n",
            "    accuracy                           0.95     73772\n",
            "   macro avg       0.90      0.77      0.82     73772\n",
            "weighted avg       0.94      0.95      0.94     73772\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y7DfiLyQRMio",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 278
        },
        "outputId": "975ded56-61b6-486d-9b91-75cb6cebdc10"
      },
      "source": [
        "y_true = y_test\n",
        "y_predicted = y_pred \n",
        "fpr, tpr, _ = roc_curve(y_true, y_predicted)\n",
        "pyplot.plot(fpr, tpr, marker='.', label='Test')\n",
        "pyplot.xlabel('False Positive Rate')\n",
        "pyplot.ylabel('True Positive Rate')\n",
        "pyplot.legend()\n",
        "pyplot.show()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxc9Xnv8c+j3Yu8SRphvMnCi8S+CDA7liAhJAF6QwgE2tCQkA1IQpJbUvJKuTRpSdOkr5ByS5yES9rQQEKbxGlIaGNjzGKDbXaQbIyNsQxYi+UdLTPz3D/OkRjLsjzGOhqN5vt+vfTynGXmPMeyf8+c3++c52fujoiI5K68TAcgIiKZpUQgIpLjlAhERHKcEoGISI5TIhARyXEFmQ7gUJWXl3tVVVWmwxARySpr1qxpc/eKgbZlXSKoqqpi9erVmQ5DRCSrmNmmA21T15CISI5TIhARyXFKBCIiOS7rxggG0tPTQ3NzM52dnZkOZciVlJQwffp0CgsLMx2KiIxSoyIRNDc3U1paSlVVFWaW6XCGjLvT3t5Oc3Mzs2fPznQ4IjJKRdY1ZGb3mFmLmb10gO1mZnea2Xoze8HMTn6vx+rs7KSsrGxUJQEAM6OsrGxUXumIyMgR5RjBvcBFg2z/ADA3/Lke+JfDOdhoSwK9Rut5icihWbOpg7seWc+aTR1D/tmRdQ25+3Izqxpkl0uBf/WgDvZKM5tkZlPd/a2oYhIRyTbuzoNrmvn6f75I0p2igjzu+9QCTpk1eciOkckxgmnA5pTl5nDdfonAzK4nuGpg5syZwxLcoWhvb6ehoQGAt99+m/z8fCoqggf4nn76aYqKigZ9/7JlyygqKuLMM8+MPFYRGfn2dsd5Yn07S5u2srSpha07u/q29cSTrNzQPmoSQdrcfRGwCKCurm7EzaRTVlbGc889B8Btt93G+PHj+epXv5r2+5ctW8b48eOVCERyWHPHXh5pamFJUwtPvtZOdzzJ+OICzptXQXXFOBYt30A8kaSwII8F1WVDeuxMJoItwIyU5enhumGxZlMHKze0s6C6bEgza9/nr1nDzTffzO7duykvL+fee+9l6tSp3Hnnndx9990UFBRw9NFHc8cdd3D33XeTn5/Pz3/+c374wx9yzjnnDHk8IjKyJJLOc5s7WNLYwtKmFpre3gXArLKxXHP6LC6ojVFXNYWigmAo9/z5scjarEwmgsXADWZ2P3A6sGMoxgf+z+9e5pU3dw66z67OHpre3kXSIc+g5ohSSksOfJ/+0UdO4G8+fEzaMbg7N954I7/97W+pqKjggQce4NZbb+Wee+7hjjvuYOPGjRQXF7N9+3YmTZrEZz/72UO+ihCR7LOzs4fH1rWxpGkry9a2sm1PN/l5xqlVk7n14lrqa2NUl48b8CaRU2ZNjuRLK0SYCMzsF8D5QLmZNQN/AxQCuPvdwEPAxcB6YC/wl1HF0t/OzjjJsIMp6cHyYIngUHV1dfHSSy9x4YUXApBIJJg6dSoAxx9/PFdffTWXXXYZl1122ZAdU0RGpo1te1jSGPT1P71xG/GkM2lsIQvnx6iviXHuvAomjsnsA6NR3jV01UG2O/CFoT5uOt/c12zq4OqfrKQnHvS3/eDKk4Y007o7xxxzDCtWrNhv2+9//3uWL1/O7373O7797W/z4osvDtlxRSTzehJJVr2+jaVhl8+Gtj0AzK8s5dPnVtNQE+OkmZPJzxs5t4ZnxWDxUDtl1mTu+9SCyPrbiouLaW1tZcWKFZxxxhn09PSwbt06amtr2bx5MwsXLuTss8/m/vvvZ/fu3ZSWlrJz5+DdWSIycm3b082ytcFA7/K1rezqilOUn8cZR5Vx7VlVLJwfY8aUsZkO84ByMhFAtP1teXl5PPjgg9x0003s2LGDeDzOl770JebNm8c111zDjh07cHduuukmJk2axIc//GEuv/xyfvvb32qwWCQLuDtrt+7qG+h95o0O3KGitJgPHj+V+poYZ80pZ1xxdjSxFvTQZI+6ujrvPzFNY2MjtbW1GYooeqP9/ESyQWdPghUb2vu6fLZsfweA46ZNpL4mxgW1lRxz5ATyRlCXTyozW+PudQNty450JSKSAVt3dvbd2//4q22805NgTGE+Z88t58b6OSysiVE5oSTTYR42JQIRkVAy6bz05o6+Lp8Xt+wAYNqkMXy0bjr1NTEWVJdRUpif4UiH1qhJBO4+Kgu0ZVvXnUi22dMV5/H1bUGXz9oWWnd1kWdw8szJ/O+L5tNQU8m8yvGjsn3pNSoSQUlJCe3t7aOuFHXvfAQlJdl/6SkykmzetpelYZfPytfa6U4kKS0Jyjk01MY4b16MKeMGrxE2moyKRDB9+nSam5tpbW3NdChDrneGMhF57+KJJM9u3h52+Wxl3dbdAFSXj+MTZ86ivqaSuqrJFObn5uy9oyIRFBYWagYvEdnHjr09PPpqK0sbt7JsXSvb9/ZQkGecNnsKV9TNoL4mRnXF+EyHOSKMikQgIuLubGjbw9LGFpY0bWXV6x0kks6UcUXU18RoqKnknHnlTBjCcjKjhRKBiGSt7nhQzqG3y+f19r1AUEjys+dVU19TyYkzJo2ocg4jkRKBiGSVtt1dLFvbytKmrSxf18burjhFBXmcdVQZ151TTX1NjGmTxmQ6zKyiRCAiI5q70/jWLpY2bWVJUwvPbd6OO1ROKObDJxxJQ02MM+eUMbZIzdl7pb85ERlxOnsSrHitnSVNW1na2MKbOzoBOGHGJL58wTzqa2Icc+SEUXW7eCYpEYjIiPD2jk6WNgV9/Y+vb6OzJ8nYonzOmVvOly6Yx/k1FcRK9UxNFJQIRCQjkknnhS07WNoYdPm8HM4sOH3yGK48dSb1NTFOr55CccHoKucwEikRiMiw2d0V5/FXW1nS2MIja1to291NnkHdrCnc8oEaGmpizImN7nIOI5ESgYhE6o32vUFff1MLKze005NwJpQUcP78WFjOoYJJY3OnnMNIpEQgIkMqnkiyZlNHXy2f9S1BOYejKsbxl2fNpqEmximzJlOQo+UcRiIlAhE5bNv3dvPouqDL59F1rex4p4fCfOP02WV8/LSgv7+qfFymw5QDUCIQkUPm7rzWupsljcG3/jWbgnIOZeOKuPDoShpqYpw9t5xSlXPICkoEIpKWrniCpzdu65u05Y1tQTmHo6dO4PPnH0V9TYwTpk8asVM1yoEpEYjIAbXu6uKRtS0sbWzhsVdb2dOdoLggj7PnlPOZ86pZOD/GkSrnkPWUCESkj7vz8ps7+wZ6n9+8HYCpE0u47KRpNNTGOKO6nDFFurd/NFEiEMlx73QneGJ9G0uaWnikqYW3d3ZiBidMn8RXLpxHQ20ltVNLdW//KKZEIJKD3tz+TljOoYUn1rfRFU8yriifc+dVUF8T4/z5MSpKizMdpgwTJQKRHJBIOs83bw8nbWmh8a2gnMPMKWP5+Okzaaip5LTZUygq0L39uUiJQGSU2tXZw2OvtrGksYVla1to39NNfp5RN2syf31xDfU1lRxVMU5dPqJEIDKavN62hyVhBc+nN26jJ+FMHFPIwvkV1NdWct7cCiaO1b39si8lApEs1pNIsvr1jr5JWza07gFgXuV4rju7mobaGCfNmKRyDjIoJQKRLNOxp5tl61r6yjns6oxTlJ/H6dVT+IsFs2iorWTGlLGZDlOySKSJwMwuAn4A5AM/cfc7+m2fCfwMmBTuc4u7PxRlTCLZxt15tWV33wTtazZ1kHQoH1/MB449gvqaSs6eW874Yn2vk/cmsn85ZpYP3AVcCDQDq8xssbu/krLbN4Bfuvu/mNnRwENAVVQxiWSLzp4ET23c1jdpS3PHOwAcO20CN9TPpaEmxnHTJqqcgwyJKL9CnAasd/cNAGZ2P3ApkJoIHJgQvp4IvBlhPCIjWsvOTh5ZG3T5PL6+jb3dCUoK8zh7TgVfWDiHhfNjHDFRUzXK0IsyEUwDNqcsNwOn99vnNuC/zexGYBxwwUAfZGbXA9cDzJw5c8gDFcmEZDIo59A7acsLzTsAOHJiCR85eTr1tTHOqC6jpFDlHCRame5UvAq4192/Z2ZnAP9mZse6ezJ1J3dfBCwCqKur8wzEKTIk9nbHefzVtr6nelt2dWEGJ82YxNfeP5+G2hjzK1XOQYZXlIlgCzAjZXl6uC7VdcBFAO6+wsxKgHKgJcK4RIZVc8feoIhbYwsrNrTTHU9SWlyQUs6hgrLxKucgmRNlIlgFzDWz2QQJ4Erg4/32eQNoAO41s1qgBGiNMCaRyCWSznObO/rq9je9vQuAqrKx/PmCWTTUxKirUjkHGTkiSwTuHjezG4CHCW4NvcfdXzaz24HV7r4Y+ArwYzP7MsHA8bXurq4fyTo7O3tYvq6VpY0tPLK2hY69PRTkGadWTeEbH6ylviZGdcX4TIcpMqBIxwjCZwIe6rfumymvXwHOijIGkahsaN3d1+Wz6vVtxJPO5LGFLJwfo742xjlzK5g4RuUcZOTL9GCxSNboSSRZtXFbWMunhY1tQTmHmiNKuf7coJzDiTMmk697+yXLKBGIDGLbnm6WrQ1KNy9f28qurjhFBXmceVQZnzyrioU1MaZPVjkHyW5KBCIp3J21W3f1DfQ+80YH7lBRWswHj59KfU2Ms+aUM07lHGQU0b9myXmdPQlWbGhnadj4b9kelHM4fvpEvtgwl4aaSo45coLKOciopUQgOWnrzs6+gd4n1rfxTk+CsUX5nD2nnJsagnIOsQkq5yC5QYlAckIy6by4ZUffpC0vbQmmapw2aQxX1E2nvraS02dPUTkHyUlKBDJq7emK89irbSxt2srSplbadneRZ3DKrMn81UU1NNTGmBsbr3IOkvOUCGRU2bwtKOfwp8atPLVhG92JJKUlBZw3r4KG2hjnzYsxZVxRpsMUGVGUCCSrxRNJnt28vW/SlnVbdwNQXTGOT5w5i/qaSuqqJlOoqRpFDkiJQLLOjr09PPpqK0sbt7JsXSvbw3IOp1dP4WOnzqS+Jsbs8nGZDlMka6SdCMxsrLvvjTIYkYG4O6+17gkmaG9sYfWmDhJJZ8q4IhpqKmmojXH23HImlKicg8h7cdBEYGZnAj8BxgMzzewE4DPu/vmog5Pc1R1P8vTGbX2TtmxqD76D1E6dwOfOO4r62hgnTJ+kcg4iQyCdK4J/At4PLAZw9+fN7NxIo5Kc1La7i0fCOj6PvdrG7q44xQV5nDWnnE+fU83CmhjTJo3JdJgio05aXUPuvrnfLXaJaMKRXOLuNL61K+jyaWrhuc3bcYfKCcV8+IQjaQjLOYwp0r39IlFKJxFsDruH3MwKgS8CjdGGJaNVZ0+CJ19r66vl89aOTgBOmDGJL18wj/qaGMccOUH39osMo3QSwWeBHxBMRr8F+G9A4wOStrd2vBPM0dvYwhOvtdHZk2RcUT7nzK3gyxcGUzXGSlXOQSRT0kkE89396tQVZnYW8EQ0IUm2Syad55u399XyeeWtoJzDjCljuPLUmTTUxjht9hSKC9TlIzISpJMIfgicnMY6yWG7Ont4/NU2ljS1sGxtC227u8kzqKuawtc/EJRzOKpC5RxERqIDJgIzOwM4E6gws5tTNk0gmINYctym9j1Bl09TCys3tNOTcCaUFHD+/FhYzqGCSWNVzkFkpBvsiqCI4NmBAqA0Zf1O4PIog5KRKZ5IsmZTR9Dl09TC+pagnMOc2Hg+edZs6mtinDJrMgUq5yCSVQ6YCNz9UeBRM7vX3TcNY0wygmzf282j61pZ0hh0+ezsjFOYbyyoLuPq04NyDrPKVM5BJJulM0aw18y+CxwD9N3a4e71kUUlGePurG/ZHdTtb2xh9aZtJB3Kxxfx/mOOCMs5VDBeUzWKjBrp/G++D3gA+BDBraSfAFqjDEqGV1c8wVMbtoVdPlvZvC2YqvGYIydww8I51NdWcvy0iZqqUWSUSicRlLn7T83siyndRauiDkyi1bKrk2VNrSxp2spjr7axtztBSWEeZ88p53PnzWFhTQVTJ6qcg0guSCcR9IR/vmVmHwTeBKZEF5JEwd15+c2d4b39W3m+eQcAUyeW8GcnTeOC2krOOKpMUzWK5KB0EsG3zGwi8BWC5wcmAF+KNCoZEu90J3hifVvfPL1bd3ZhBifOmMRX3zeP+ppKaqeW6t5+kRx30ETg7v8VvtwBLIS+J4tlBNqyvbecw1aefK2drniS8cUFnDuvnPqaSs6fX0H5+OJMhykiI8hgD5TlA1cQ1Bj6o7u/ZGYfAv4aGAOcNDwhymASSee5zdv7Jm1pensXALPKxnL16bNoqI1xatUUigp0b7+IDGywK4KfAjOAp4E7zexNoA64xd1/MxzBycB2dvbw2Lo2ljRtZdnaVrbt6SY/zzi1ajK3XlxLfW2M6vJx6vIRkbQMlgjqgOPdPWlmJcDbwFHu3j48oUmq19v2sCQc6H164zbiSWfS2EIWzo9RXxPj3LkVTByrqRpF5NANlgi63T0J4O6dZrbhUJOAmV1EUMI6H/iJu98xwD5XALcBDjzv7h8/lGOMVj2JJKtf7+ibtGVD6x4A5lWO51PnVNNQG+OkGZNUzkFEDttgiaDGzF4IXxtwVLhsgLv78YN9cDjGcBdwIdAMrDKzxe7+Sso+c4GvA2e5e4eZxQ7jXLJex55ulq0LSjc/uq6VXZ1xivLzWHBUGZ84o4r6mhgzpozNdJgiMsoMlghqD/OzTwPWu/sGADO7H7gUeCVln08Dd7l7B4C7txzmMbOKu7Nu6+5ggvbGFp55o4OkQ0VpMRcfO5X62hhnzylnnMo5iEiEBis6d7iF5qYBm1OWm4HT++0zD8DMniDoPrrN3f/Y/4PM7HrgeoCZM2ceZliZ1dmTYOWG9r5JW7ZsD8o5HDdtIjfWz6WhNsaxR6qcg4gMn0x/1SwA5gLnA9OB5WZ2nLtvT93J3RcBiwDq6up8uIM8XC07O/tKNz/+ahvv9CQYU5jP2XPLubF+DgtrYlRO0FSNIpIZUSaCLQS3n/aaHq5L1Qw85e49wEYzW0eQGLK6llEyGZRz+FPjVpY2tfDilqCcw7RJY7j8lOnU18Y4o1rlHERkZEgrEZjZGGCmu689hM9eBcw1s9kECeBKoP8dQb8BrgL+n5mVE3QVbTiEY4wYe7vjPP5qW9+MXS27gnIOJ8+czNfeP5+G2hjzK1XOQURGnoMmAjP7MPCPBDOWzTazE4Hb3f2Swd7n7nEzuwF4mKD//x53f9nMbgdWu/vicNv7zOwVIAF8LZueU9i8bS+PrA36+ldsaKc7nqS0uIBz51fQUBPj/PkxpozTVI0iMrKZ++Bd7ma2BqgHlrn7SeG6F939uGGIbz91dXW+evXqTByaRNJ59o2Ovklb1m4NyjlUl4+jviZGfVjOoVD39ovICGNma9y9bqBtaZWhdvcd/bo0sm7A9r3a8U4Py9e1srQpmKqxY28PBXnGabOn8I26WuprYlRXjM90mCIi71k6ieBlM/s4kB8+AHYT8GS0YWXWa627WdoYzNa16vUOEklnyrgiFtbEaKip5Jx55UwoUTkHERkd0kkENwK3Al3AvxP0638ryqCGW3c8yerXt4V1+1vY2BaUc6g5opTPnBuUczhxxmTydW+/iIxC6SSCGne/lSAZjAprNgU1fPLM2NC6h+XrWtnVFaeoII8zjyrjk2dVsbAmxvTJKucgIqNfOonge2Z2BPAg8IC7vxRxTJFas6mDKxetoCcRDHNMHlvIh06YSn1NJWfNKWNsUaafsRMRGV7pzFC2MEwEVwA/MrMJBAkhK7uHVm5oJx4mgTyDT50zmy8snJvhqEREMiet+xzd/W13vxP4LPAc8M1Io4rQguqyvjo+RQV5LKguz3BEIiKZddBEYGa1Znabmb1IMHn9kwTlIrLSKbMm8/5jKikqyOO+Ty3glFmTMx2SiEhGpdMhfg/wAPB+d38z4niGxaSxRUwoKVASEBEhvTGCM4YjkOHUHU9SpKd/RUSAQRKBmf3S3a8Iu4RSnyROa4aykaw7nqSoQIlARAQGvyL4Yvjnh4YjkOGkRCAi8q4Dtobu/lb48vPuvin1B/j88IQXje6EEoGISK90WsMLB1j3gaEOZDhpjEBE5F2DjRF8juCbf7WZvZCyqRR4IurAoqSuIRGRdw02RvDvwB+AvwduSVm/y923RRpVxLoSSSYWqXqoiAgMngjc3V83sy/032BmU7I5GahrSETkXQe7IvgQsIbg9tHUGswOVEcYV6S64wmK1TUkIgIMkgjc/UPhn7OHL5zhobuGRETelU6tobPMbFz4+hoz+76ZzYw+tOh09ahrSESkVzqt4b8Ae83sBOArwGvAv0UaVcR0RSAi8q50WsO4uztwKfDP7n4XwS2kWUu3j4qIvCud6qO7zOzrwJ8D55hZHpDV914qEYiIvCud1vBjBBPXf9Ld3yaYi+C7kUYVoWTSiSdddw2JiIQO2hqGjf99wEQz+xDQ6e7/GnlkEelOJAF0RSAiEkrnrqErgKeBjxLMW/yUmV0edWBR6YqHiUB3DYmIAOmNEdwKnOruLQBmVgH8CXgwysCi0h0mAnUNiYgE0mkN83qTQKg9zfeNSOoaEhHZVzpXBH80s4eBX4TLHwMeii6kaPVeESgRiIgE0pmz+Gtm9r+As8NVi9z919GGFZ2+RJCfn+FIRERGhsHmI5gL/CNwFPAi8FV33zJcgUVFVwQiIvsarDW8B/gv4CMEFUh/eKgfbmYXmdlaM1tvZrcMst9HzMzNrO5Qj3GouhMJQIlARKTXYF1Dpe7+4/D1WjN75lA+2MzygbsIprpsBlaZ2WJ3f6XffqXAF4GnDuXz3yvdPioisq/BEkGJmZ3Eu/MQjElddveDJYbTgPXuvgHAzO4nqFf0Sr/9/hb4DvC1Q4z9PVHXkIjIvgZLBG8B309Zfjtl2YH6g3z2NGBzynIzcHrqDmZ2MjDD3X9vZgdMBGZ2PXA9wMyZh1cBW88RiIjsa7CJaRZGeeCweN33gWsPtq+7LwIWAdTV1fnhHFfPEYiI7CvK1nALMCNleXq4rlcpcCywzMxeBxYAi6MeMO7WGIGIyD6ibA1XAXPNbLaZFQFXAot7N7r7Dncvd/cqd68CVgKXuPvqCGPSGIGISD+RtYbuHgduAB4GGoFfuvvLZna7mV0S1XEPRl1DIiL7OuiTxWZmwNVAtbvfHs5XfIS7P32w97r7Q/QrR+Hu3zzAvuenFfFh0hWBiMi+0mkN/y9wBnBVuLyL4PmArKTnCERE9pVO0bnT3f1kM3sWwN07wj7/rKTBYhGRfaXTGvaETwk79M1HkIw0qgh1J5IU5ht5eXbwnUVEckA6ieBO4NdAzMy+DTwO/F2kUUWoO57U1YCISIp0ylDfZ2ZrgAaC8hKXuXtj5JFFpDue1ECxiEiKdO4amgnsBX6Xus7d34gysKgoEYiI7CudweLfE4wPGFACzAbWAsdEGFdkuhNKBCIiqdLpGjoudTksFPf5yCKKmMYIRET2dcgtYlh++vSD7jhCdcWTFBVomkoRkV7pjBHcnLKYB5wMvBlZRBFT15CIyL7SGSMoTXkdJxgz+I9owoledzxBsbqGRET6DJoIwgfJSt39q8MUT+S640nGFqWT/0REcsMBvxqbWYG7J4CzhjGeyKlrSERkX4N9NX6aYDzgOTNbDPwK2NO70d3/M+LYIqG7hkRE9pVOH0kJ0E4wR3Hv8wQOZG8i0BWBiEifwRJBLLxj6CXeTQC9Dmve4ExSIhAR2ddgiSAfGM++CaBX9iYCjRGIiOxjsETwlrvfPmyRDJOuHo0RiIikGqxFHJUF+7sSSYp1RSAi0mewFrFh2KIYJu6uMQIRkX4O2CK6+7bhDGQ49CSCoQ11DYmIvCunWsTuRDhfsa4IRET65FSL2DdxvRKBiEifnGoRlQhERPaXUy1ibyIo1nwEIiJ9cisRJBKArghERFLlVIvY1ds1pLuGRET65FSL+G7XUE6dtojIoHKqRdRgsYjI/nKqRdRzBCIi+4u0RTSzi8xsrZmtN7NbBth+s5m9YmYvmNkSM5sVZTzdGiMQEdlPZC1iON/xXcAHgKOBq8zs6H67PQvUufvxwIPAP0QVD6hrSERkIFG2iKcB6919g7t3A/cDl6bu4O6PuPvecHElMD3CeNQ1JCIygChbxGnA5pTl5nDdgVwH/GGgDWZ2vZmtNrPVra2t7zkg3T4qIrK/EdEimtk1QB3w3YG2u/sid69z97qKior3fBzdPioisr90Jq9/r7YAM1KWp4fr9mFmFwC3Aue5e1eE8WiMQERkAFG2iKuAuWY228yKgCuBxak7mNlJwI+AS9y9JcJYAI0RiIgMJLIW0d3jwA3Aw0Aj8Et3f9nMbjezS8LdvguMB35lZs+Z2eIDfNyQ0O2jIiL7i7JrCHd/CHio37pvpry+IMrj99cdT5JnUKBEICLSJ6daxO6E5isWEekvp1rF7nhS3UIiIv3kVKvYFU9SpElpRET2kVOJoDue1DMEIiL95FSrqDECEZH95VSr2B1PaIxARKSfnGoVu+O6IhAR6S+nWkV1DYmI7C+nWkXdPioisr+cahXVNSQisr+cahW7lAhERPaTU62ixghERPaXU61idzxJscYIRET2kVOtosYIRET2l1OtorqGRET2l1Otom4fFRHZX061irprSERkfznTKiaSTiLpSgQiIv3kTKvYN1+xEoGIyD5yplXUxPUiIgPLmVaxK5EA0MQ0IiL95EyrqK4hEZGB5UyrqEQgIjKwnGkVuxO9YwSavF5EJFXuJAJdEYiIDChnWkUlAhGRgeVMq9ibCHTXkIjIvnKmVexK6IpARGQgOdMq6oEyEZGB5UyrqK4hEZGB5UyrqMFiEZGBRdoqmtlFZrbWzNab2S0DbC82swfC7U+ZWVVUsXRrjEBEZECRtYpmlg/cBXwAOBq4ysyO7rfbdUCHu88B/gn4TlTxvNayG4Cmt3ZGdQgRkawU5dfj04D17r7B3buB+4FL++1zKfCz8PWDQIOZ2VAHsmZTB/c++ToAn/v5M6zZ1DHUhxARyVpRJoJpwOaU5eZw3YD7uHsc2AGU9f8gM7vezFab2erW1tZDDr+wcUMAAAjSSURBVGTlhnYSSQegJ5Fk5Yb2Q/4MEZHRKis6zN19kbvXuXtdRUXFIb9/QXUZxYV55BsUFuSxoHq/XCMikrMKIvzsLcCMlOXp4bqB9mk2swJgIjDkX9dPmTWZ+z61gJUb2llQXcYpsyYP9SFERLJWlIlgFTDXzGYTNPhXAh/vt89i4BPACuByYKm7exTBnDJrshKAiMgAIksE7h43sxuAh4F84B53f9nMbgdWu/ti4KfAv5nZemAbQbIQEZFhFOUVAe7+EPBQv3XfTHndCXw0yhhERGRwWTFYLCIi0VEiEBHJcUoEIiI5TolARCTHWUR3a0bGzFqBTe/x7eVA2xCGkw10zrlB55wbDuecZ7n7gE/kZl0iOBxmttrd6zIdx3DSOecGnXNuiOqc1TUkIpLjlAhERHJcriWCRZkOIAN0zrlB55wbIjnnnBojEBGR/eXaFYGIiPSjRCAikuNGZSIws4vMbK2ZrTezWwbYXmxmD4TbnzKzquGPcmilcc43m9krZvaCmS0xs1mZiHMoHeycU/b7iJm5mWX9rYbpnLOZXRH+rl82s38f7hiHWhr/tmea2SNm9mz47/viTMQ5VMzsHjNrMbOXDrDdzOzO8O/jBTM7+bAP6u6j6oeg5PVrQDVQBDwPHN1vn88Dd4evrwQeyHTcw3DOC4Gx4evP5cI5h/uVAsuBlUBdpuMeht/zXOBZYHK4HMt03MNwzouAz4WvjwZez3Tch3nO5wInAy8dYPvFwB8AAxYATx3uMUfjFcFpwHp33+Du3cD9wKX99rkU+Fn4+kGgwcxsGGMcagc9Z3d/xN33hosrCWaMy2bp/J4B/hb4DtA5nMFFJJ1z/jRwl7t3ALh7yzDHONTSOWcHJoSvJwJvDmN8Q87dlxPMz3IglwL/6oGVwCQzm3o4xxyNiWAasDlluTlcN+A+7h4HdgDZPJFxOuec6jqCbxTZ7KDnHF4yz3D33w9nYBFK5/c8D5hnZk+Y2Uozu2jYootGOud8G3CNmTUTzH9y4/CEljGH+v/9oCKdmEZGHjO7BqgDzst0LFEyszzg+8C1GQ5luBUQdA+dT3DVt9zMjnP37RmNKlpXAfe6+/fM7AyCWQ+PdfdkpgPLFqPximALMCNleXq4bsB9zKyA4HKyfViii0Y654yZXQDcClzi7l3DFFtUDnbOpcCxwDIze52gL3Vxlg8Yp/N7bgYWu3uPu28E1hEkhmyVzjlfB/wSwN1XACUExdlGq7T+vx+K0ZgIVgFzzWy2mRURDAYv7rfPYuAT4evLgaUejsJkqYOes5mdBPyIIAlke78xHOSc3X2Hu5e7e5W7VxGMi1zi7qszE+6QSOff9m8IrgYws3KCrqINwxnkEEvnnN8AGgDMrJYgEbQOa5TDazHwF+HdQwuAHe7+1uF84KjrGnL3uJndADxMcMfBPe7+spndDqx298XATwkuH9cTDMpcmbmID1+a5/xdYDzwq3Bc/A13vyRjQR+mNM95VEnznB8G3mdmrwAJ4GvunrVXu2me81eAH5vZlwkGjq/N5i92ZvYLgmReHo57/A1QCODudxOMg1wMrAf2An952MfM4r8vEREZAqOxa0hERA6BEoGISI5TIhARyXFKBCIiOU6JQEQkxykRyIhkZgkzey7lp2qQfXcPwfHuNbON4bGeCZ9QPdTP+ImZHR2+/ut+25483BjDz+n9e3nJzH5nZpMOsv+J2V6NU6Kn20dlRDKz3e4+fqj3HeQz7gX+y90fNLP3Af/o7scfxucddkwH+1wz+xmwzt2/Pcj+1xJUXb1hqGOR0UNXBJIVzGx8OI/CM2b2opntV2nUzKaa2fKUb8znhOvfZ2Yrwvf+yswO1kAvB+aE7705/KyXzOxL4bpxZvZ7M3s+XP+xcP0yM6szszuAMWEc94Xbdod/3m9mH0yJ+V4zu9zM8s3su2a2Kqwx/5k0/lpWEBYbM7PTwnN81syeNLP54ZO4twMfC2P5WBj7PWb2dLjvQBVbJddkuva2fvQz0A/BU7HPhT+/JngKfkK4rZzgqcreK9rd4Z9fAW4NX+cT1BsqJ2jYx4Xr/wr45gDHuxe4PHz9UeAp4BTgRWAcwVPZLwMnAR8Bfpzy3onhn8sI5zzojSlln94Y/wz4Wfi6iKCK5BjgeuAb4fpiYDUwe4A4d6ec36+Ai8LlCUBB+PoC4D/C19cC/5zy/r8DrglfTyKoRTQu079v/WT2Z9SVmJBR4x13P7F3wcwKgb8zs3OBJME34Urg7ZT3rALuCff9jbs/Z2bnEUxW8kRYWqOI4Jv0QL5rZt8gqFNzHUH9ml+7+54whv8EzgH+CHzPzL5D0J302CGc1x+AH5hZMXARsNzd3wm7o443s8vD/SYSFIvb2O/9Y8zsufD8G4H/Sdn/Z2Y2l6DMQuEBjv8+4BIz+2q4XALMDD9LcpQSgWSLq4EK4BR377GgomhJ6g7uvjxMFB8E7jWz7wMdwP+4+1VpHONr7v5g74KZNQy0k7uvs2Cug4uBb5nZEne/PZ2TcPdOM1sGvB/4GMFEKxDMNnWjuz98kI94x91PNLOxBPV3vgDcSTABzyPu/mfhwPqyA7zfgI+4+9p04pXcoDECyRYTgZYwCSwE9ptz2YJ5mLe6+4+BnxBM97cSOMvMevv8x5nZvDSP+RhwmZmNNbNxBN06j5nZkcBed/85QTG/geaM7QmvTAbyAEGhsN6rCwga9c/1vsfM5oXHHJAHs83dBHzF3i2l3luK+NqUXXcRdJH1ehi40cLLIwuq0kqOUyKQbHEfUGdmLwJ/ATQNsM/5wPNm9izBt+0fuHsrQcP4CzN7gaBbqCadA7r7MwRjB08TjBn8xN2fBY4Dng67aP4G+NYAb18EvNA7WNzPfxNMDPQnD6ZfhCBxvQI8Y8Gk5T/iIFfsYSwvEEzM8g/A34fnnvq+R4CjeweLCa4cCsPYXg6XJcfp9lERkRynKwIRkRynRCAikuOUCEREcpwSgYhIjlMiEBHJcUoEIiI5TolARCTH/X8wBB6Xm1igiAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JoY6nWYFg17I",
        "colab_type": "text"
      },
      "source": [
        "### Бейзлайн на даунсэмпле (для ансамблей и нейронных сетей)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i8MRaAsTPyov",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.utils import resample\n",
        "\n",
        "data_majority = data[data.toxic==0]\n",
        "data_minority = data[data.toxic==1]\n",
        "length = len(data_minority)\n",
        "\n",
        "data_majority_downsampled = resample(data_majority, \n",
        "                                 replace=False,    \n",
        "                                 n_samples=length,     \n",
        "                                 random_state=123) \n",
        "\n",
        "data_downsampled = pd.concat([data_majority_downsampled, data_minority])"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RDHm8yzyg5wH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = data_downsampled['comment_text']\n",
        "y = data_downsampled['toxic']"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KK2O2Qilg8vu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nSNcVZ6Sg_Oq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "count_vect = CountVectorizer()"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UL0gexVdhB4v",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0c1e35b0-ade5-42fd-bb0b-19fcdf57cb0f"
      },
      "source": [
        "X_train = count_vect.fit_transform(X_train)\n",
        "X_train.shape"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(28654, 64225)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tr1RpUg_hCL3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "72c880f7-40af-4126-870c-258532642523"
      },
      "source": [
        "X_test = count_vect.transform(X_test)\n",
        "X_test.shape"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(14114, 64225)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4l_og_cuhCaL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "clf = SGDClassifier().fit(X_train, y_train)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MGtEC89mhF2K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_pred = clf.predict(X_test)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "prApC6yRhHr-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 176
        },
        "outputId": "e25318e7-e6ad-4616-f37d-985e987e7743"
      },
      "source": [
        "print(classification_report(y_test, y_pred))"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.89      0.89      0.89      7072\n",
            "           1       0.89      0.89      0.89      7042\n",
            "\n",
            "    accuracy                           0.89     14114\n",
            "   macro avg       0.89      0.89      0.89     14114\n",
            "weighted avg       0.89      0.89      0.89     14114\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dq3iYVLfSGEf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 278
        },
        "outputId": "73d44f26-3635-4c24-c4a5-0c9e4a68bdd0"
      },
      "source": [
        "y_true = y_test\n",
        "y_predicted = y_pred \n",
        "fpr, tpr, _ = roc_curve(y_true, y_predicted)\n",
        "pyplot.plot(fpr, tpr, marker='.', label='Test')\n",
        "pyplot.xlabel('False Positive Rate')\n",
        "pyplot.ylabel('True Positive Rate')\n",
        "pyplot.legend()\n",
        "pyplot.show()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5RddX338fcnk5lMJheSTCJg7lggCYoCUwQvVYoXRARaqULhaWltebwA9UFdDy0upLS2WFq7xNIiKgtbUUBaJRaU3qD0USIkgFwyA00DyMAMxCHJTDJM5pLv88feJzmZzOXEzD63/XmtNWvO3meffb47A/u79+/329+fIgIzM8uvaZUOwMzMKsuJwMws55wIzMxyzonAzCznnAjMzHJueqUDOFALFy6MFStWVDoMM7OasmHDhp9HxKKx3qu5RLBixQrWr19f6TDMzGqKpOfGe89NQ2ZmOedEYGaWc04EZmY5V3N9BGMZGhqis7OTgYGBSocy5Zqbm1myZAmNjY2VDsXM6lRdJILOzk7mzJnDihUrkFTpcKZMRNDT00NnZycrV66sdDhmVqcyaxqSdJOklyU9Mc77knSdpE2SHpN0/C/6XQMDA7S2ttZVEgCQRGtra13e6ZhZ9ciyj+Bm4LQJ3n8fcGT6cxHwdwfzZfWWBArq9bjM7MBseG4r19+7iQ3PbZ3yfWfWNBQR90taMcEmZwF/H0kd7HWS5kk6PCK6sorJzKzaDQ7vZmv/IK/sHGTrzkF6dg7y085t3PyjZ9kdQdP0adzyeydxwvL5U/adlewjWAw8X7Tcma7bLxFIuojkroFly5aVJbgD0dPTw6mnngpAd3c3DQ0NLFqUPMD34IMP0tTUNOHn77vvPpqamnjLW96SeaxmVj4RQe/AMK/sHNzzs3XnIK/0D+6z7pWdg8nJf8cgfbuGJ9zn0PBu1m3uqZtEULKIuBG4EaCtra3qZtJpbW3l0UcfBeCqq65i9uzZfPrTny758/fddx+zZ892IjCrcruGR4pO6EP07NyVnNjTk/vedUP07BxkW/8gw7vHPmU1TZ9G66wmFqQ/yxa07Hld+Jnf0kTr7Caef6WfT3zrYYaGd9M4fRonHdE6pcdVyUTwArC0aHlJuq4sNjy3lXWbezjpiNYpzax79r9hA5dddhk7duxg4cKF3HzzzRx++OFcd9113HDDDUyfPp01a9ZwzTXXcMMNN9DQ0MA3v/lNvvzlL/P2t799yuMxs33t3h30DiQn7EITzJ6r9R17r9qL1+0cHBlzXxLMm9m45wS+vLWF45fPY35L0Ul9VhOtRSf3mY0NJfcBHnXoHG75vZMyO2dVMhGsBS6WdCvwZmD7VPQP/PH3n2Tji70TbtM3MERHdx+7A6YJVh02hznN44/TX/PauXzuA8eUHENEcMkll3DnnXeyaNEibrvtNq644gpuuukmrrnmGp555hlmzJjBtm3bmDdvHh/96EcP+C7CzPY1MDSy92Q++qd/1Mk+bYoZ52KdmY0N+5zAj1g0Oz2pN7Jg1ox9fs9vaWJeSxMN07Id2HHC8vmZXLRCholA0reBdwILJXUCnwMaASLiBuBu4HRgE9AP/E5WsYzWOzC85z+A3ZEsT5QIDtSuXbt44oknePe73w3AyMgIhx9+OADHHnss559/PmeffTZnn332lH2nWT0Z2R1s60/bzXcO8crOXfv83to/uN9J/9Whsa/Wp4k9V+bzZzXxukWz+eWVTSwY62p9VrJ+ZlNDmY+4srIcNXTeJO8H8Imp/t5Srtw3PLeV87+2bk9725fOPW5KM21EcMwxx/DAAw/s995dd93F/fffz/e//30+//nP8/jjj0/Z95pVo4jg1aERenYMjnkC39o/uOe9wrptrw4R41ytz2pqYMHs5ITdOruJIw+dnZzU03WjT+qHzGxkWsZX67WuJjqLp9oJy+dn2t42Y8YMtmzZwgMPPMDJJ5/M0NAQTz/9NKtXr+b555/nlFNO4W1vexu33norO3bsYM6cOfT2TtycZVYthkd2s7V/aJ8T936jX0at2zW8e8x9NUxT0mY+q4n5sxpZddhc5s9q3HO1npzUZyTr0vb15sZ8Xa2XQy4TAWTb3jZt2jTuuOMOLr30UrZv387w8DCf/OQnOeqoo7jgggvYvn07EcGll17KvHnz+MAHPsA555zDnXfe6c5iK6uIYMeuYbbuHEo7R0c1wRTa1Yva2Le/OjTu/ubMmM6C2ckJ+9C5zaw+fO7eUTDp1XrxqJi5zdP90GQVUIx3/1Wl2traYvTENO3t7axevbpCEWWv3o/Pps7QyO79Rr4Ud5L27Ny33X3rziEGR8a+Wm9s0J6r8NHDGvcMbZy19+Q+v6WJpukuaFytJG2IiLax3svtHYFZtSs8jDTukMaiETGF130D4z+MNLd5Oq2zZzC/pZHF82byhsVz9xvSWHzSnz3DV+t54URgVia7hkeSJpgJhjTu6TxN1032MFLhxL10fss+I2AWjLqKn9fSSGODr9ZtbHWTCCKiLq9eaq3pLi8KDyON7iTde1JP29n79zbB7BindEDhYaTCCXzZghbetHTePlfrhRExhRN7S1PpDyOZTaYuEkFzczM9PT11V4q6MB9Bc3NzpUOpewNDI/uf1HeMPQJma/8gW/uHGBnnar25cVrRSJcZrGxt2WdIY+uodvdDZjYy3VfrVkF1kQiWLFlCZ2cnW7ZsqXQoU64wQ5mVbvfuYNur+1+tj3VCL5zs+8cpHVB4GKlwtf66RbP3HadeeMK06Ko9bw8jWe2ri0TQ2NjoGbzqWP/g8L5Fvva5Wt/b9JK8N8S2CUoHtDQ17NN2/kvpiX28ETFzZzZmXjrArNLqIhFY7RjZHfuMSR9rRExxSd5X+gcZGJr4YaQF6cNGRx82Z78hjcUVHP0wktnYnAjsFxYR7BwcGXPky+hx7IV12ycoHTBnxnTmpyfx18xp5uhD5xYNaRxV7KuliTnN0106wGwKOBHYHkMju9Or9b011ccq8lV8JT84TumAxgbt0yG6+rVz9xvSWHy1Pq+lkRnTfbVuVglOBHUqIujbNTzh1froDtTeSR5GKpy8F89r5vWvnTtmka/C7zl+GMmsZjgR1IjCPKZjDWkcr6Lj0Mg4DyM1TNun9O7i+S1FQxqTppf5sxr3DIGc39Lkh5HM6pgTQZkUz4h23NJ59A0Mj13kq+hkXzi5b9058Tym81r2VmtcuqCFNy6Zt98DSMVX67P8MJKZFXEiKIMNz23lvK+u29OePk2MO7xxRqF0QHoCX9Hask/lxtFNMPP8MJKZHSQngjJYt7mHoTQJCDhx5QLetfrQPSNiiuuttzT5T2Jm5eWzThmcdEQrEkTAjMZpfOa9qzKbC8HM7EA5EZTB8cvmMaupgRULZ3PVmcc4CZhZVXHjchm83LeLvl0j/Prxi50EzKzqOBGUQXtXMh/x6sPnVjgSM7P9ORGUQXtXHwCrD3MiMLPq40RQBh3dvbz2kGYOaWmsdChmZvtxIiiD9q5eNwuZWdVyIsjYwNAI/7NlJ6sOn1PpUMzMxuREkLFNL+9gZHf4jsDMqpYTQcYKI4ZWuaPYzKqUE0HGOrr7mDF9GisXzqp0KGZmY3IiyFh7Vy9HHzbH896aWdVyIshQRCQjhtwsZGZVzIkgQy/37WJr/xCrPWLIzKpYpolA0mmSnpK0SdLlY7y/TNK9kh6R9Jik07OMp9w2FjqKPWLIzKpYZolAUgNwPfA+YA1wnqQ1ozb7LHB7RBwHnAv8bVbxVEKHS0uYWQ3I8o7gRGBTRGyOiEHgVuCsUdsEUDhLHgK8mGE8Zdfe5dISZlb9skwEi4Hni5Y703XFrgIukNQJ3A1cMtaOJF0kab2k9Vu2bMki1kx0dLu0hJlVv0p3Fp8H3BwRS4DTgX+QtF9MEXFjRLRFRNuiRYvKHuQvolBawonAzKpdlongBWBp0fKSdF2xjwC3A0TEA0AzsDDDmMqmUFrCNYbMrNplmQgeAo6UtFJSE0ln8NpR2/wMOBVA0mqSRFA7bT8T8GQ0ZlYrMksEETEMXAzcA7STjA56UtLVks5MN/sU8PuSfgp8G7gwIiKrmMqpvauP5sZprGh1aQkzq26ZTl4fEXeTdAIXr7uy6PVG4K1ZxlApHd29HH2oS0uYWfWrdGdxXSqUlnDFUTOrBU4EGXBpCTOrJU4EGdjojmIzqyFOBBnwZDRmVkucCDLQ0dXH4nkzXVrCzGqCE0EGko5i9w+YWW1wIphiA0MjbP65S0uYWe1wIphiLi1hZrXGiWCKecSQmdUaJ4Ip1uHSEmZWY0pOBJJasgykXrR3ubSEmdWWSROBpLdI2gh0pMtvlFRXU0pOlYjwZDRmVnNKuSP4a+C9QA9ARPwU+JUsg6pVL/UmpSU8dNTMaklJTUMR8fyoVSMZxFLz2rvdUWxmtaeUMtTPS3oLEJIagT8gmV/ARtlTWsKJwMxqSCl3BB8FPkEy8fwLwJuAj2cZVK1qL5SWmOnSEmZWO0q5Izg6Is4vXiHprcCPsgmpdnV09br0tJnVnFLuCL5c4rpcK5SWcMVRM6s1494RSDoZeAuwSNJlRW/NBRqyDqzWFEpLuKPYzGrNRE1DTcDsdJvi9o5e4Jwsg6pFG/d0FLtpyMxqy7iJICL+E/hPSTdHxHNljKkmubSEmdWqUjqL+yVdCxwDNBdWRsSvZhZVDWrv6uXow+a6tISZ1ZxSOotvISkvsRL4Y+BZ4KEMY6o5EUF7dy+r/USxmdWgUhJBa0R8HRiKiP+MiN8FfDdQ5KXeXWzrH3JHsZnVpFKahobS312S3g+8CCzILqTas3eyet8RmFntKSUR/KmkQ4BPkTw/MBf4ZKZR1ZhCjSGXljCzWjRpIoiIf05fbgdOgT1PFlvKpSXMrJZN9EBZA/AhkhpDP4yIJySdAfwRMBM4rjwhVr92l5Ywsxo20R3B14GlwIPAdZJeBNqAyyPie+UIrhYMDI2wecsO3vf6wyodipnZL2SiRNAGHBsRuyU1A93A6yKipzyh1Yb/fmkHuwPXGDKzmjXR8NHBiNgNEBEDwOYDTQKSTpP0lKRNki4fZ5sPSdoo6UlJ3zqQ/VeDvZPRuGnIzGrTRHcEqyQ9lr4W8Lp0WUBExLET7TjtY7geeDfQCTwkaW1EbCza5kjgD4G3RsRWSa85iGOpiPauXpobp7HcpSXMrEZNlAhWH+S+TwQ2RcRmAEm3AmcBG4u2+X3g+ojYChARLx/kd5ZdR1efS0uYWU2bqOjcwRaaWwwUz3XcCbx51DZHAUj6EUlp66si4oejdyTpIuAigGXLlh1kWFOnUFrCHcVmVstKmrw+Q9OBI4F3AucBX5U0b/RGEXFjRLRFRNuiRYvKHOL4unsH2NY/5I5iM6tpWSaCF0iGnxYsSdcV6wTWRsRQRDwDPE2SGGpCR1cfgGsMmVlNKykRSJop6egD3PdDwJGSVkpqAs4F1o7a5nskdwNIWkjSVLT5AL+nYgqT0RztGkNmVsMmTQSSPgA8CvwwXX6TpNEn9P1ExDBwMXAP0A7cHhFPSrpa0pnpZvcAPZI2AvcCn6ml5xQ6ul1awsxqXylF564iGQF0H0BEPCppZSk7j4i7gbtHrbuy6HUAl6U/NScpLeFmITOrbaU0DQ1FxPZR6yKLYGpJobSEHyQzs1pXyh3Bk5J+E2hIHwC7FPhxtmFVv0JpCd8RmFmtK+WO4BKS+Yp3Ad8iKUed+/kIPBmNmdWLUu4IVkXEFcAVWQdTS9q7e5nZ2ODSEmZW80q5I/grSe2S/kTS6zOPqEa0d/Vy1GFzXFrCzGrepIkgIk4hmZlsC/AVSY9L+mzmkVWxiKCju4817ig2szpQ0gNlEdEdEdcBHyV5puDKST5S1wqlJdxRbGb1oJQHylZLukrS4yST1/+YpFxEbu3tKHYiMLPaV0pn8U3AbcB7I+LFjOOpCe1pjaFVbhoyszowaSKIiJPLEUgtae/qZfG8mcxtdmkJM6t94yYCSbdHxIfSJqHiJ4lLmqGsnnV097l/wMzqxkR3BH+Q/j6jHIHUikJpCU9GY2b1YtzO4ojoSl9+PCKeK/4BPl6e8KqPS0uYWb0pZfjou8dY976pDqRWFEYMORGYWb2YqI/gYyRX/kdIeqzorTnAj7IOrFpt7EpKSyxb0FLpUMzMpsREfQTfAn4A/DlwedH6voh4JdOoqlhHdy9Hu7SEmdWRiZqGIiKeBT4B9BX9IGlB9qFVn4igvavPcxCYWV2Z7I7gDGADyfDR4kvgAI7IMK6q1N07wPZXXVrCzOrLuIkgIs5If5c0LWUeuKPYzOpRKbWG3ippVvr6AklflLQs+9CqT6G0xNGejMbM6kgpw0f/DuiX9EbgU8D/AP+QaVRVqr2rlyXzXVrCzOpLKYlgOCICOAv4m4i4nmQIae60d/W64qiZ1Z1SEkGfpD8E/hdwl6RpQO4uiQeGRnjm5zs9GY2Z1Z1SEsGHSSau/92I6CaZi+DaTKOqQk+/1MfugFXuKDazOlPKVJXdwC3AIZLOAAYi4u8zj6zKdKQdxR4xZGb1ppRRQx8CHgR+A/gQ8BNJ52QdWLUplJZY7tISZlZnSpmh7ArglyPiZQBJi4B/A+7IMrBq096VlJaY5tISZlZnSukjmFZIAqmeEj9XNyLCk9GYWd0q5Y7gh5LuAb6dLn8YuDu7kKpP1/ZCaQmPGDKz+lPKnMWfkfTrwNvSVTdGxHezDau6dHS7tISZ1a+J5iM4EvhL4HXA48CnI+KFcgVWTVxawszq2URt/TcB/wx8kKQC6ZcPdOeSTpP0lKRNki6fYLsPSgpJbQf6HeXg0hJmVs8mahqaExFfTV8/JenhA9mxpAbgepKpLjuBhyStjYiNo7abA/wB8JMD2X85tXf1ulnIzOrWRHcEzZKOk3S8pOOBmaOWJ3MisCkiNkfEIHArSb2i0f4E+AIwcMDRl0GhtMRqNwuZWZ2a6I6gC/hi0XJ30XIAvzrJvhcDzxctdwJvLt4gTShLI+IuSZ8Zb0eSLgIuAli2rLwVsAulJXxHYGb1aqKJaU7J8ovT4nVfBC6cbNuIuBG4EaCtrS2yjGu0wmQ0rjFkZvUqywfDXgCWFi0vSdcVzAFeD9wn6VngJGBttXUYt3f10dLk0hJmVr+yTAQPAUdKWimpCTgXWFt4MyK2R8TCiFgRESuAdcCZEbE+w5gOmEtLmFm9yywRRMQwcDFwD9AO3B4RT0q6WtKZWX3vVCqUlvBkNGZWzyZ9sliSgPOBIyLi6nS+4sMi4sHJPhsRdzOqHEVEXDnOtu8sKeIyKpSW8GQ0ZlbPSrkj+FvgZOC8dLmP5PmAuueOYjPLg1KKzr05Io6X9AhARGxN2/zrXke3S0uYWf0r5Y5gKH1KOGDPfAS7M42qSmx0aQkzy4FSEsF1wHeB10j6PPD/gD/LNKoq0eHSEmaWA6WUob5F0gbgVEDA2RHRnnlkFVYoLfH+Y19b6VDMzDJVyqihZUA/8P3idRHxsywDq7SnutPSEu4fMLM6V0pn8V0k/QMCmoGVwFPAMRnGVXGejMbM8qKUpqE3FC+nheI+nllEVaJQWmKZS0uYWZ074CeLI+JhRlURrUcuLWFmeVFKH8FlRYvTgOOBFzOLqApEBO1dvZzxRncUm1n9K6WPoLi3dJikz+AfswmnOnRtH6B3YNgdxWaWCxMmgvRBsjkR8ekyxVMVCqUl3FFsZnkwbh+BpOkRMQK8tYzxVIVCInBpCTPLg4nuCB4k6Q94VNJa4DvAzsKbEfFPGcdWMe3dfSxdMJM5Li1hZjlQSh9BM9BDMkdx4XmCAOo3EXT1eg4CM8uNiRLBa9IRQ0+wNwEUlHXe4HJ6dXCEZ3++kzNcWsLMcmKiRNAAzGbfBFBQt4ng6ZeS0hKejMbM8mKiRNAVEVeXLZIqUSgt4aYhM8uLiZ4szuUjtS4tYWZ5M1EiOLVsUVSRjS4tYWY5M24iiIhXyhlINYgIT0ZjZrlzwEXn6tmLhdISTgRmliNOBEU6CqUl/ESxmeWIE0ERl5YwszxyIijS3uXSEmaWP04ERdq7e1nt5wfMLGecCFKF0hKr3FFsZjnjRJByaQkzyysngpQnozGzvHIiSHV09zGrqYGl811awszyJdNEIOk0SU9J2iTp8jHev0zSRkmPSfp3ScuzjGciLi1hZnmVWSJI5zu+HngfsAY4T9KaUZs9ArRFxLHAHcBfZBXPRCIimYzGzUJmlkNZ3hGcCGyKiM0RMQjcCpxVvEFE3BsR/eniOmBJhvGM68XtA/S5tISZ5VSWiWAx8HzRcme6bjwfAX4w1huSLpK0XtL6LVu2TGGIifYXk45ijxgyszyqis5iSRcAbcC1Y70fETdGRFtEtC1atGjKv78wGc3RfpjMzHKolMnrf1EvAEuLlpek6/Yh6V3AFcA7ImJXhvGMq72rj2ULWpg9I8t/DjOz6pTlHcFDwJGSVkpqAs4F1hZvIOk44CvAmRHxcoaxTKi9u5dVLjRnZjmVWSKIiGHgYuAeoB24PSKelHS1pDPTza4FZgPfkfSopLXj7C4zhdIS7ig2s7zKtC0kIu4G7h617sqi1+/K8vtL8VRaWmK1O4rNLKeqorO4kjpcWsLMci73iaC9q9elJcws15wIuvtcWsLMci3XiaBQWsLNQmaWZ7lOBC9se5W+gWHXGDKzXMt1Iujo6gNcWsLM8i3XiaAwGY1LS5hZnuU6EXR0u7SEmVmuE0HSUexmITPLt9wmglcHR3imZyer3CxkZjmX20Tw1Et9RPiJYjOz3CaC9j2lJdw0ZGb5lttE0OHSEmZmQI4TQXtXH6sOn+vSEmaWe7lMBBHhyWjMzFK5TASF0hLuKDYzy2kiKJSWcEexmVlOE4FLS5iZ7ZXPRNDd69ISZmapXCaCjq4+NwuZmaVylwj6B4d5pmenO4rNzFK5SwRPv7SDCFxjyMwslbtEUOgoXuM7AjMzIKeJYFZTA0vmz6x0KGZmVSF3iaDDpSXMzPaRq0RQKC3hEUNmZnvlKhEUSku4o9jMbK9cJYL2PaUlnAjMzApylQg69pSWcNOQmVlBrhJBe3cvy1tdWsLMrFi+EkFXn+cgMDMbJdNEIOk0SU9J2iTp8jHenyHptvT9n0hakVUs/YPDPOvSEmZm+8ksEUhqAK4H3gesAc6TtGbUZh8BtkbELwF/DXwhq3jufPRFIqBpeq5ugszMJpXlWfFEYFNEbI6IQeBW4KxR25wFfCN9fQdwqqQpf9Jrw3NbufLOJwD40r/9Nxue2zrVX2FmVrOyTASLgeeLljvTdWNuExHDwHagdfSOJF0kab2k9Vu2bDngQNZt7mF4JAAYHtnNus09B7wPM7N6VRPtJBFxY0S0RUTbokWLDvjzJx3RyozGaTQIGqdP46Qj9ss1Zma5leU4yheApUXLS9J1Y23TKWk6cAgw5ZfrJyyfzy2/dxLrNvdw0hGtnLB8/lR/hZlZzcoyETwEHClpJckJ/1zgN0dtsxb4beAB4BzgPyIisgjmhOXznQDMzMaQWSKIiGFJFwP3AA3ATRHxpKSrgfURsRb4OvAPkjYBr5AkCzMzK6NMH7GNiLuBu0etu7Lo9QDwG1nGYGZmE6uJzmIzM8uOE4GZWc45EZiZ5ZwTgZlZzimj0ZqZkbQFeO4X/PhC4OdTGE4t8DHng485Hw7mmJdHxJhP5NZcIjgYktZHRFul4ygnH3M++JjzIatjdtOQmVnOORGYmeVc3hLBjZUOoAJ8zPngY86HTI45V30EZma2v7zdEZiZ2ShOBGZmOVeXiUDSaZKekrRJ0uVjvD9D0m3p+z+RtKL8UU6tEo75MkkbJT0m6d8lLa9EnFNpsmMu2u6DkkJSzQ81LOWYJX0o/Vs/Kelb5Y5xqpXw3/YySfdKeiT97/v0SsQ5VSTdJOllSU+M874kXZf+ezwm6fiD/tKIqKsfkpLX/wMcATQBPwXWjNrm48AN6etzgdsqHXcZjvkUoCV9/bE8HHO63RzgfmAd0FbpuMvwdz4SeASYny6/ptJxl+GYbwQ+lr5eAzxb6bgP8ph/BTgeeGKc908HfgAIOAn4ycF+Zz3eEZwIbIqIzRExCNwKnDVqm7OAb6Sv7wBOlaQyxjjVJj3miLg3IvrTxXUkM8bVslL+zgB/AnwBGChncBkp5Zh/H7g+IrYCRMTLZY5xqpVyzAHMTV8fArxYxvimXETcTzI/y3jOAv4+EuuAeZIOP5jvrMdEsBh4vmi5M1035jYRMQxsB2p5IuNSjrnYR0iuKGrZpMec3jIvjYi7yhlYhkr5Ox8FHCXpR5LWSTqtbNFlo5Rjvgq4QFInyfwnl5QntIo50P/fJ5XpxDRWfSRdALQB76h0LFmSNA34InBhhUMpt+kkzUPvJLnru1/SGyJiW0WjytZ5wM0R8VeSTiaZ9fD1EbG70oHVinq8I3gBWFq0vCRdN+Y2kqaT3E72lCW6bJRyzEh6F3AFcGZE7CpTbFmZ7JjnAK8H7pP0LElb6toa7zAu5e/cCayNiKGIeAZ4miQx1KpSjvkjwO0AEfEA0ExSnK1elfT/+4Gox0TwEHCkpJWSmkg6g9eO2mYt8Nvp63OA/4i0F6ZGTXrMko4DvkKSBGq93RgmOeaI2B4RCyNiRUSsIOkXOTMi1lcm3ClRyn/b3yO5G0DSQpKmos3lDHKKlXLMPwNOBZC0miQRbClrlOW1FvitdPTQScD2iOg6mB3WXdNQRAxLuhi4h2TEwU0R8aSkq4H1EbEW+DrJ7eMmkk6ZcysX8cEr8ZivBWYD30n7xX8WEWdWLOiDVOIx15USj/ke4D2SNgIjwGciombvdks85k8BX5X0f0g6ji+s5Qs7Sd8mSeYL036PzwGNABFxA0k/yOnAJqAf+J2D/s4a/vcyM7MpUI9NQ2ZmdgCcCMzMcs6JwMws55wIzMxyzonAzCznnAisKkkakfRo0c+KCbbdMQXfd7OkZ9Lvejh9QvVA9/E1SWvS13806r0fH2yM6X4K/y5PSPq+pHmTbP+mWq/Gadnz8FGrSpJ2RMTsqd52gn3cDPxzRNwh6T3AX0bEsQexv4OOabL9SvoG8HREfH6C7S8kqbp68VTHYvXDdyuTuF4AAAODSURBVARWEyTNTudReFjS45L2qzQq6XBJ9xddMb89Xf8eSQ+kn/2OpMlO0PcDv5R+9rJ0X09I+mS6bpakuyT9NF3/4XT9fZLaJF0DzEzjuCV9b0f6+1ZJ7y+K+WZJ50hqkHStpIfSGvP/u4R/lgdIi41JOjE9xkck/VjS0emTuFcDH05j+XAa+02SHky3Hatiq+VNpWtv+8c/Y/2QPBX7aPrzXZKn4Oem7y0keaqycEe7I/39KeCK9HUDSb2hhSQn9lnp+v8LXDnG990MnJO+/g3gJ8AJwOPALJKnsp8EjgM+CHy16LOHpL/vI53zoBBT0TaFGH8N+Eb6uomkiuRM4CLgs+n6GcB6YOUYce4oOr7vAKely3OB6enrdwH/mL6+EPibos//GXBB+noeSS2iWZX+e/unsj91V2LC6sarEfGmwoKkRuDPJP0KsJvkSvhQoLvoMw8BN6Xbfi8iHpX0DpLJSn6UltZoIrmSHsu1kj5LUqfmIyT1a74bETvTGP4JeDvwQ+CvJH2BpDnpvw7guH4AfEnSDOA04P6IeDVtjjpW0jnpdoeQFIt7ZtTnZ0p6ND3+duBfi7b/hqQjScosNI7z/e8BzpT06XS5GViW7styyonAasX5wCLghIgYUlJRtLl4g4i4P00U7wdulvRFYCvwrxFxXgnf8ZmIuKOwIOnUsTaKiKeVzHVwOvCnkv49Iq4u5SAiYkDSfcB7gQ+TTLQCyWxTl0TEPZPs4tWIeJOkFpL6O58AriOZgOfeiPi1tGP9vnE+L+CDEfFUKfFaPriPwGrFIcDLaRI4BdhvzmUl8zC/FBFfBb5GMt3fOuCtkgpt/rMkHVXid/4XcLakFkmzSJp1/kvSa4H+iPgmSTG/seaMHUrvTMZyG0mhsMLdBSQn9Y8VPiPpqPQ7xxTJbHOXAp/S3lLqhVLEFxZt2kfSRFZwD3CJ0tsjJVVpLeecCKxW3AK0SXoc+C2gY4xt3gn8VNIjJFfbX4qILSQnxm9LeoykWWhVKV8YEQ+T9B08SNJn8LWIeAR4A/Bg2kTzOeBPx/j4jcBjhc7iUf6FZGKgf4tk+kVIEtdG4GElk5Z/hUnu2NNYHiOZmOUvgD9Pj734c/cCawqdxSR3Do1pbE+my5ZzHj5qZpZzviMwM8s5JwIzs5xzIjAzyzknAjOznHMiMDPLOScCM7OccyIwM8u5/w+R1qIButdb3AAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mk03LEJ19yRR",
        "colab_type": "text"
      },
      "source": [
        "### Top features Eli5"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GH8uW_PQSL41",
        "colab_type": "text"
      },
      "source": [
        "На всех данных"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bH-cdUPXDo-X",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "outputId": "05de2b96-3bea-47a4-a9ac-cc258d04372d"
      },
      "source": [
        "eli5.formatters.as_dataframe.explain_weights_df(clf)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target</th>\n",
              "      <th>feature</th>\n",
              "      <th>weight</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>x70089</td>\n",
              "      <td>2.116671</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>x84835</td>\n",
              "      <td>2.067145</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>x64497</td>\n",
              "      <td>2.009007</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>x20704</td>\n",
              "      <td>1.970248</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>x69133</td>\n",
              "      <td>1.912109</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>71457</th>\n",
              "      <td>1</td>\n",
              "      <td>x23850</td>\n",
              "      <td>-1.195068</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>71458</th>\n",
              "      <td>1</td>\n",
              "      <td>x69507</td>\n",
              "      <td>-1.287659</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>71459</th>\n",
              "      <td>1</td>\n",
              "      <td>x166952</td>\n",
              "      <td>-1.511600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>71460</th>\n",
              "      <td>1</td>\n",
              "      <td>x122363</td>\n",
              "      <td>-2.082218</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>71461</th>\n",
              "      <td>1</td>\n",
              "      <td>x71831</td>\n",
              "      <td>-2.377217</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>71462 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       target  feature    weight\n",
              "0           1   x70089  2.116671\n",
              "1           1   x84835  2.067145\n",
              "2           1   x64497  2.009007\n",
              "3           1   x20704  1.970248\n",
              "4           1   x69133  1.912109\n",
              "...       ...      ...       ...\n",
              "71457       1   x23850 -1.195068\n",
              "71458       1   x69507 -1.287659\n",
              "71459       1  x166952 -1.511600\n",
              "71460       1  x122363 -2.082218\n",
              "71461       1   x71831 -2.377217\n",
              "\n",
              "[71462 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6o7n1LI95Qsf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def analyze_features(model, n):\n",
        "  index_to_word = {v:k for k,v in count_vect.vocabulary_.items()}\n",
        "  data = eli5.formatters.as_dataframe.explain_weights_df(model)\n",
        "  \n",
        "  class1 = data[data['target']==1]\n",
        "  \n",
        "  print('\\n', n, 'самых значимых признаков класса (toxic):')\n",
        "  for element in class1.feature[:n]:\n",
        "    feature = element.strip('x')\n",
        "    print(index_to_word[int(feature)], element)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7tDYpjAa5q0O",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 561
        },
        "outputId": "e8e81ab0-c1ad-4188-b816-2a4d028dc2ad"
      },
      "source": [
        "analyze_features(clf, 30)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            " 30 самых значимых признаков класса (toxic):\n",
            "fuck x70089\n",
            "idiot x84835\n",
            "faggot x64497\n",
            "asshole x20704\n",
            "france x69133\n",
            "fucking x70128\n",
            "idiots x84845\n",
            "suck x164233\n",
            "shit x154929\n",
            "bitch x28022\n",
            "fucked x70099\n",
            "stupid x163563\n",
            "cunt x46526\n",
            "moron x114373\n",
            "bark x24314\n",
            "niggers x119867\n",
            "ass x20589\n",
            "sucks x164261\n",
            "motherfucker x114630\n",
            "bullshit x32375\n",
            "bastard x24691\n",
            "penis x129840\n",
            "nigger x119855\n",
            "jerk x91987\n",
            "fuckin x70126\n",
            "dumbass x56450\n",
            "cock x40275\n",
            "aidsaids x13669\n",
            "fag x64461\n",
            "nigga x119839\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cpV-lJ9JWaRB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "64af5fa5-ace7-4866-e42f-4856b2fcc9b3"
      },
      "source": [
        "sum(sum(clf.coef_ != 0))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "71461"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wfe0px_MVwwx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "top_features = [int(i[1:]) for i in eli5.formatters.as_dataframe.explain_weights_df(clf).feature if 'BIAS' not in i]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gpv-y9HWV6qO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b4291a44-2e94-4e80-a274-ba8cbd22cf6c"
      },
      "source": [
        "len(top_features)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "71461"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 74
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r9-Hk2SDrCnq",
        "colab_type": "text"
      },
      "source": [
        "### Top features without BIAS"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JsyTu9DvrHct",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train_eli5 = X_train[:,top_features]\n",
        "X_test_eli5 = X_test[:,top_features]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N06mhC_RrWWf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "eli5_model = SGDClassifier().fit(X_train_eli5, y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kFh6HxQerZhi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_pred = eli5_model.predict(X_test_eli5)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bVx9XtiNre-w",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "7175ad05-cf4f-4752-a22d-8c5577ef6ddf"
      },
      "source": [
        "print(classification_report(y_test, y_pred))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.98      0.97     66765\n",
            "           1       0.80      0.61      0.70      7007\n",
            "\n",
            "    accuracy                           0.95     73772\n",
            "   macro avg       0.88      0.80      0.83     73772\n",
            "weighted avg       0.95      0.95      0.95     73772\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0qSTqoZySkrR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "outputId": "b94a78b7-b830-4452-f1e6-d1b5e1e0c2a1"
      },
      "source": [
        "y_true = y_test\n",
        "y_predicted = y_pred \n",
        "fpr, tpr, _ = roc_curve(y_true, y_predicted)\n",
        "pyplot.plot(fpr, tpr, marker='.', label='Test')\n",
        "pyplot.xlabel('False Positive Rate')\n",
        "pyplot.ylabel('True Positive Rate')\n",
        "pyplot.legend()\n",
        "pyplot.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3RddZ338fc3915yaU9CaZumuVik5SK0gTYFRMQLolx8QAXhGXRkGFRARVyPM7iUB8cZHEZnicMziMrCGVFQZtQ6osxNhhlJ6AVKoUWwTVuaXmh7mqYtSZrL+T5/7J3T0zRNT2n2OTk5n9daWT17n332/u6m3d/z+/32/v7M3RERkfxVkO0AREQku5QIRETynBKBiEieUyIQEclzSgQiInmuKNsBHK/q6mqvr6/PdhgiIjll1apVu929ZqT3ci4R1NfXs3LlymyHISKSU8xs89HeU9eQiEieUyIQEclzSgQiInku58YIRtLf309HRwe9vb3ZDmXMlZWVUVtbS3FxcbZDEZEJakIkgo6ODsrLy6mvr8fMsh3OmHF34vE4HR0dNDQ0ZDscEZmgIusaMrOHzGynmb10lPfNzO4zs/VmtsbMFr7ZY/X29hKLxSZUEgAwM2Kx2IRs6YjI+BHlGMHDwCWjvP8+YF74cxPw9ydysImWBIZM1PMSkeOzanMn9/92Pas2d475viPrGnL3p82sfpRNrgD+wYM62G1mVmVmM919e1QxiYjkmr6BBD9ZsYW7frmWhDslRQU8cuMSFs2dNmbHyOYYwWxgS8pyR7juiERgZjcRtBqoq6vLSHDHIx6Pc/HFFwOwY8cOCgsLqakJHuBbvnw5JSUlo37+qaeeoqSkhKVLl0Yeq4iMb/2DCV7c2kXrhjitG+Ks3LyH3v7EofcHErS1xydMIkibuz8IPAjQ3Nw87mbSicVirF69GoC77rqLqVOncscdd6T9+aeeeoqpU6cqEYjkocGEs3ZbeOFvj7Ni4x7e6BsE4NSTy7nmnDpOqijlW//+BwYGExQXFbCkMTamMWQzEWwF5qQs14brMmLV5k7a2uMsaYyNaWZN7n/VKm6//XYOHDhAdXU1Dz/8MDNnzuS+++7jgQceoKioiAULFnDPPffwwAMPUFhYyA9/+EO+/e1vc8EFF4x5PCIyPiQSzu937Ke1PfjG/+zGOPt7BwBoqpnCBxfOpqWxmiWN04lNLU1+bnFDLLJrVjYTwTLgFjN7FFgMdI3F+MD//eVa1m3bN+o2+3v7+f2O/SQcCizIuuVlR79Pf8GsCr5y2Wlpx+Du3HrrrfziF7+gpqaGxx57jDvvvJOHHnqIe+65h40bN1JaWsrevXupqqri5ptvPu5WhIjkBndn/c4DyQt/W3uczu5+AObGJvP+M2bS0hSjpTHGSRVlR93PornTIvnSChEmAjP7MfAOoNrMOoCvAMUA7v4A8ARwKbAe6AY+HlUsw+3rHSARdjAlPFgeLREcr4MHD/LSSy/x7ne/G4DBwUFmzpwJwJlnnsl1113HlVdeyZVXXjlmxxSR8cHd2RTvTnb1tG6Is/vAQQBmV03i4vkzaGmM0dIUY1bVpCxHG4jyrqFrj/G+A58e6+Om88191eZOrvteG/0DQX/bt645e0wzrbtz2mmn0draesR7v/rVr3j66af55S9/yde+9jVefPHFMTuuiGTHlj3dyYt+64Y4O/YFz/7MqCjl/LfEwm/81cyZPmlc3hKeE4PFY23R3Gk8cuOSyPrbSktL2bVrF62trbS0tNDf38+rr77K/Pnz2bJlCxdddBHnn38+jz76KAcOHKC8vJx9+0bvzhKR8WN7V0/yot/aHqejsweA2JQSloTdPC1NMRqrp4zLC/9weZkIINr+toKCAh5//HFuu+02urq6GBgY4LOf/SynnHIK119/PV1dXbg7t912G1VVVVx22WVcffXV/OIXv9Bgscg4tGv/wcP6+DfufgOAyknFLGmczo3nN7D0LdXMO2lqTlz4h7OghyZ3NDc3+/CJaV5++WXmz5+fpYiiN9HPT2S86Xyjj7b2Q338f9h5AIDy0iLObZgedPU0xZh/cgUFBblx4TezVe7ePNJ7edsiEBEZ0tXTz/KNe5JdPS9vD7pqJ5cU0lw/nasW1dLSGOO0WRUUFU686v1KBCKSdw4cHGDFpj20bYjzzIY4a7d1kXAoLSqguX4ad7znFFqaYpxZW0XxBLzwDzdhEoG752Tf3LHkWtedyHjU0zfIqs2dPLNhN63tcdZ0dDGYcEoKCzirropb3zmPlqYYZ9dVUVpUmO1wM25CJIKysjLi8fiEK0U9NB9BWdnRHzIRkSP19g/y/Gt7aW2P07YhzvNbOukfdIoKjDNrK7n5wkZaGqtZNHcak0ry78I/3IRIBLW1tXR0dLBr165shzLmhmYoE5Gj6xtIsKZjb7KPf9XmTg4OJCgwOH12JX98XgMtTTHOqZ/OlNIJcdkbUxPib6S4uFgzeInkkYHBBC9t23dYobae/qBQ2/yZFVy3eC5Lm2Kc0zCdykma5vVYJkQiEJGJLZFw1m3fF9zSuSHO8o172H8wKNQ276SpfLi5lpamGIsbYkybMnrZdzmSEoGIjDvuzquvH6B1w26e2RDn2Y176OoJCrU1Vk/hsrNm0dIYY0ljjJry0mPsTY5FiUBEss7dad/9Bs9sCAZ329rjxN/oA2DO9Em897QZyXo9J1fq5omxpkQgIhnn7ry25/AKnTv3BxU6Z1aWceEpNcmaPXOmT85ytBOfEoGIZMTWvYcKtbW1x9m6NyjUVj21NFmPf2lTjLmxyRPqNvBcoEQgIpHYua/3UGnm9jib490ATJtczJLGWHAvf1OMpprcLNQ2kSgRiMiYiB84SFv7HlrbgwHe9l1Bhc7ysiKWNMa4oaWelqYYb51RnjOF2vKFEoGIvCld3f20bTw0Gcsrr+8HYEpJIec2TOeac+bQ0ljNglkVFOrCP64pEYhIWvb3Hl6hc932fbhDWXEB59RP5/KzZrG0KcYZsysnZIXOiUyJQERG1N03wIpNnckL/0tbw0JtRQUsrKvic+8KKnS+rbaKkiJd+HOZEoGIAEGhtuc2dyYHeF/o2Jss1HbWnCo+/Y4mljTFWFg3jbJiFWqbSJQIRPJU30CC1VuGCrXt5rnX9tI3kKCwwDhjdiU3XtBIS2OM5vppTC7RpWIi029XJE8MDCZYs7UrObi7cvMeevsTmMFpsyq4oWVuskJneZkKteUTJQKRCWow4azbti85GcuKjXt4oy+o0HnqyeVcc04dLU0xljTEqJysC38+UyIQmSASCeeV1/fzzIahCp1x9vUGFTqbaqbwwYWzWdpUzeKG6cSmqlCbHKJEIJKj3J31Ow8kB3fb2uN0dgcVOufGJnPpGTOTpRtOqlChNjk6JQKRHOHubIofXqht94GgUNvsqklcPH8GLY0xWppizKqalOVoJZcoEYiMY1v2dCfn3X1mQ5wd+3oBmFFRyvlviSVLM8+ZPkn1euRNUyIQGUd2dPUGtXrWB9/6OzqDCp2xKSXJsswtTTEaq6fowi9jRolAJIt27T94WB//xt1BobbKScUsaZzOjec3sPQt1cw7SRU6JTpKBCIZ1PlGXzDvbnjx/8POAwCUlxZxbsN0rlsc3NI5/+QKVeiUjFEiEIlQV8/hhdpe3r4PgMklhTTXT+eqRbW0NMY4bVaFCrVJ1kSaCMzsEuBbQCHwPXe/Z9j7dcAPgKpwmy+6+xNRxiQSpQMHB1ixaQ9tKYXaEg6lRQU010/jjvcEhdrOrK2iWBd+GSciSwRmVgjcD7wb6ABWmNkyd1+XstmXgJ+4+9+b2QLgCaA+qphExlpP3yCrNncmJ2NZ0xFW6Cws4Ky6Km595zxammKcXVdFaZEKtcn4FGWL4Fxgvbu3A5jZo8AVQGoicKAifF0JbIswHpETdnBgkOdf28szG4JbOldv2UvfYIKiAuPM2spg+sXGahbNncakEl34JTdEmQhmA1tSljuAxcO2uQv4VzO7FZgCvGukHZnZTcBNAHV1dWMeqMjR9A8mWNOxN3k756rNnRwcSFBgcPrsSj5+Xn2yUNuUUg25SW7K9r/ca4GH3f0bZtYC/KOZne7uidSN3P1B4EGA5uZmz0KckicGBhO8tG1fcnB35aY9dIeF2ubPrOC6xXNZ2hTjnIbpVE5SoTaZGKJMBFuBOSnLteG6VJ8ALgFw91YzKwOqgZ0RxiWSlEg467bvC27p3BBn+cY97D8YFGqbd9JUPrSolpamGIsbYkybUpLlaEWiEWUiWAHMM7MGggRwDfDRYdu8BlwMPGxm84EyYFeEMUmec3deff0ArWFp5rb2PXT1BIXaGquncNlZs2hpjLGkMUZNuSp0Sn6ILBG4+4CZ3QI8SXBr6EPuvtbM7gZWuvsy4PPAd83scwQDxx9zd3X9yJhxd9p3v5GcjKWtPU78jT4A5kyfxHtPm5Gs13NypSp0Sn6KdIwgfCbgiWHrvpzyeh1wXpQxSH5xd7bs6Uneztm6Ic7O/UGFzpmVZVx4Sk1w4W+KUTttcpajFRkfsj1YLHLCtu7tOewb/9a9QaG26qmlLG2KJWvyz41NVr0ekREoEUjO2bmvN1mrp7U9zuZ4NwDTJhfT0hQL7uVvitFUo0JtIulQIpBxL37gIG3te2ht303rhjgbdgUVOivKiljcGOOGluBe/rfOKFehNpE3QYlAxp2u7n7aNsaT3T2vvL4fgCklhZzbMJ2PnDOHlsZqFsyqoFAXfpETpkQgWbe/t58Vm4IKnc9siLNu+z7coay4gHPqp3P5WbNY2hTjjNmVqtApEgElAsm47r4BVm7qDO7qCSt0DiackqICFtZV8bl3BRU631ZbRUmRLvwiUVMikMj19g/y3ObO5ADvCx176R90igqMs+ZU8el3NLGkKcbCummUFatQm0imKRHImOsbSLB6y97wrp7dPPfaXvoGEhQWGGfMruTGCxppaYzRXD+NySX6JyiSbfpfKCdsYDDBmq1dyfv4V2zaQ29/AjM4bVYFN7TMTVboLC9ToTaR8UaJQI7bYMJZt21f8nbO5Rv38EZYofPUk8u55pxg3t0lDTEqJ+vCLzLepZ0IzGyyu3dHGYyMT4mE88rr+5N39SzfGGdfb1Chs6lmCh9cOJulTdUsbphObKoKtYnkmmMmAjNbCnwPmArUmdnbgD91909FHZxkh7uzYdeBZK2etvY4nd1Bhc65sclcesbMZNmGkypUqE0k16XTIvhb4L3AMgB3f8HM3h5pVJJR7s6meHeyZENbe5xdYaG22VWTuHj+DFoag5o9s6omZTlaERlraXUNufuWYTVbBqMJRzJly57u4KIfXvy3d/UCMKOilPOShdqqmTN9kur1iExw6SSCLWH3kJtZMfAZ4OVow5KxtqOrNzm4+8yGOB2dQYXO2JQSloTdPC1NMRqrp+jCL5Jn0kkENwPfIpiMfivwr4DGB8a5XfsPBtMvhg9xbdwdFGqrnFTMksbp3Hh+A0vfUs28k1ShUyTfpZMI3uru16WuMLPzgN9FE5K8GZ1v9PHsxnhygPcPOw8AUF5axLkN07lucXBL5/yTK1ShU0QOk04i+DawMI11kkFdPf0s37gnOcD7+x1BobbJJYWcUz+dqxbV0tIY47RZFSrUJiKjOmoiMLMWYClQY2a3p7xVQTAHsWTQgYMDrNi0Jzm4+9LWLhIOpUUFNNdP4/PvDgq1nVlbRbEu/CJyHEZrEZQQPDtQBJSnrN8HXB1lUAI9fYOs2tyZHOB9oSOs0FlYwFl1Vdz6znm0NMU4u66K0iLlZRF5846aCNz9v4D/MrOH3X1zBmPKSwcHBnn+tb3Jrp7Vr+2lbzBBUYFxZm0lN1/YyNKmahbWTWNSiS78IjJ20hkj6Daze4HTgORjpO7+zsiiygP9gwnWdOxN3s65anMnBwcSFBicPruSj59fT0tjUKhtSqlKQolIdNK5wjwCPAZ8gOBW0huAXVEGNRENDCZYu21fcjKWlZv20B0Walsws4Lrl8wNLvwN06mcpEJtIpI56SSCmLt/38w+k9JdtCLqwHJdIuGs274vuJc/rNC5/2BQqO2UGVP50KJaWppiLG6IMW1KSZajFZF8lk4i6A//3G5m7we2AdOjCyk3uTuvvn6A1g27aW2P8+zGPewNC7U1Vk/hsrNm0dIYY0ljjJpyVegUkfEjnUTwF2ZWCXye4PmBCuCzkUaVA9yd9t1vJAd3n22Ps/tAHwBzpk/iPQtmJOv1nFypCp0iMn4dMxG4+7+EL7uAiyD5ZHFecXe27OlJ3s7Z2h7n9X1Bhc6ZlWW8fV5NcOFvilE7bXKWoxURSd9oD5QVAh8mqDH0G3d/ycw+APw5MAk4OzMhZs+2vT3Ju3ra2uNs3RsUaqueWsrSZIXOGHNjk1WvR0Ry1mgtgu8Dc4DlwH1mtg1oBr7o7j/PRHCZtnNfb7JIW2t7nM3xYEK2aZOLaWmKcfOFjbQ0xWiqUaE2EZk4RksEzcCZ7p4wszJgB9Dk7vHMhBa9363fzWMrtnBwYJD1Ow+wYVdQobOirIjFjTFuaKmnpSnGW2eUq1CbiExYoyWCPndPALh7r5m1H28SMLNLCEpYFwLfc/d7Rtjmw8BdgAMvuPtHj+cYb9aqzZ380feXM+gOwMK6Kv780lNpaaxmwawKCnXhF5E8MVoiONXM1oSvDWgKlw1wdz9ztB2HYwz3A+8GOoAVZrbM3delbDMP+DPgPHfvNLOTTuBcjktbezyZBAoNLp4/g5ve3pSpw4uIjBujJYL5J7jvc4H17t4OYGaPAlcA61K2+RPgfnfvBHD3nSd4zLQtaYxRYJBwKC4qYEljLFOHFhEZV0YrOneiheZmA1tSljuAxcO2OQXAzH5H0H10l7v/ZviOzOwm4CaAurq6EwwrsGjuNOpjUwC490NvY9HcaWOyXxGRXJPtwvVFwDzgHcC1wHfNrGr4Ru7+oLs3u3tzTU3NmB28oMA4dWa5koCI5LUoE8FWgttPh9SG61J1AMvcvd/dNwKvEiSGjOjpG6SsWCWdRSS/pZUIzGySmb31OPe9AphnZg1mVgJcAywbts3PCVoDmFk1QVdR+3Ee503r7R9ksmr7i0ieO2YiMLPLgNXAb8Lls8xs+AX9CO4+ANwCPAm8DPzE3dea2d1mdnm42ZNA3MzWAb8FvpDJ5xR6+geZpBaBiOS5dIrO3UVwB9BTAO6+2swa0tm5uz8BPDFs3ZdTXjtwe/iTUe6uRCAiQnpdQ/3u3jVsnUcRTCYdHEjgDmXqGhKRPJdOi2CtmX0UKAwfALsNeCbasKLX2x/MDqYWgYjku3RaBLcSzFd8EPgRQTnqnJ+PoEeJQEQESK9FcKq73wncGXUwmdQTzhc8SV1DIpLn0mkRfMPMXjazr5rZ6ZFHlCFDLQI9RyAi+e6YicDdLyKYmWwX8B0ze9HMvhR5ZBHTGIGISCCtB8rcfYe73wfcTPBMwZeP8ZFxr1tdQyIiQHoPlM03s7vM7EWCyeufISgXkdOSYwRqEYhInktnsPgh4DHgve6+LeJ4MkZjBCIigWMmAndvyUQgmTY0RqBaQyKS746aCMzsJ+7+4bBLKPVJ4rRmKBvv1DUkIhIYrUXwmfDPD2QikEzr6U8AGiwWETnqYLG7bw9ffsrdN6f+AJ/KTHjRGRojKC3K9tw8IiLZlc5V8N0jrHvfWAeSab1h5VEzy3YoIiJZNdoYwScJvvk3mtmalLfKgd9FHVjUevoG1S0kIsLoYwQ/An4N/BXwxZT1+919T6RRZYDmIhARCYyWCNzdN5nZp4e/YWbTcz0Z9PQPUlas8QERkWO1CD4ArCK4fTS1M92BxgjjilyvuoZERIBREoG7fyD8M61pKXONuoZERALp1Bo6z8ymhK+vN7Nvmlld9KFFq7tvUOUlRERI7/bRvwe6zextwOeBDcA/RhpVBvSqRSAiAqSXCAbc3YErgL9z9/sJbiHNaT39g6ozJCJCetVH95vZnwH/G7jAzAqA4mjDip6eIxARCaTTIvgIwcT1f+zuOwjmIrg30qgyILh9VIlARCSdqSp3AI8AlWb2AaDX3f8h8sgipjECEZFAOncNfRhYDnwI+DDwrJldHXVgUeofTNA/6EoEIiKkN0ZwJ3COu+8EMLMa4N+Bx6MMLErJies1RiAiktYYQcFQEgjF0/zcuKVpKkVEDkmnRfAbM3sS+HG4/BHgiehCil5vXzgpjRKBiEhacxZ/wcz+F3B+uOpBd/9ZtGFFq0ddQyIiSaPNRzAP+BugCXgRuMPdt2YqsCglE4FaBCIio/b1PwT8C3AVQQXSbx/vzs3sEjN7xczWm9kXR9nuKjNzM2s+3mO8Gd19A4DGCEREYPSuoXJ3/274+hUze+54dmxmhcD9BFNddgArzGyZu68btl058Bng2ePZ/4kYumtIJSZEREZPBGVmdjaH5iGYlLrs7sdKDOcC6929HcDMHiWoV7Ru2HZfBb4OfOE4Y3/TeoYGi5UIRERGTQTbgW+mLO9IWXbgncfY92xgS8pyB7A4dQMzWwjMcfdfmdlRE4GZ3QTcBFBXd+IVsDVGICJyyGgT01wU5YHD4nXfBD52rG3d/UHgQYDm5mY/0WPrOQIRkUOifDBsKzAnZbk2XDekHDgdeMrMNgFLgGWZGDDu7dPtoyIiQ6JMBCuAeWbWYGYlwDXAsqE33b3L3avdvd7d64E24HJ3XxlhTEBKi6Aopx+QFhEZE5FdCd19ALgFeBJ4GfiJu681s7vN7PKojpuOnv5BSgoLKCpUIhAROeaTxWZmwHVAo7vfHc5XfLK7Lz/WZ939CYaVo3D3Lx9l23ekFfEY6OkbpKxYSUBEBNJrEfw/oAW4NlzeT/B8QM7q7dfsZCIiQ9IpOrfY3Rea2fMA7t4Z9vnnrB5NSiMikpROi6A/fErYITkfQSLSqCIWdA0pEYiIQHqJ4D7gZ8BJZvY14H+Av4w0qoj1qGtIRCQpnTLUj5jZKuBigvISV7r7y5FHFqGevkHVGRIRCaVz11Ad0A38MnWdu78WZWBR6ukfpGpycbbDEBEZF9IZLP4VwfiAAWVAA/AKcFqEcUWqp19jBCIiQ9LpGjojdTksFPepyCLKgN4+3TUkIjLkuJ+qCstPLz7mhuOYBotFRA5JZ4zg9pTFAmAhsC2yiDJAzxGIiBySzhhBecrrAYIxg3+KJpzoJRJOb39CYwQiIqFRE0H4IFm5u9+RoXgid3BAs5OJiKQ66hiBmRW5+yBwXgbjiZxmJxMROdxoLYLlBOMBq81sGfBT4I2hN939nyOOLRJKBCIih0tnjKAMiBPMUTz0PIEDuZkIwtnJytQ1JCICjJ4ITgrvGHqJQwlgyAnPG5wtvWoRiIgcZrREUAhM5fAEMCRnE0F32CJQrSERkcBoiWC7u9+dsUgyJDlfsVoEIiLA6E8Wj9QSyHlDYwTqGhIRCYyWCC7OWBQZlBwjUNeQiAgwSiJw9z2ZDCRTdPuoiMjhjrvoXK5T15CIyOHyLxEMDRaX5N2pi4iMKO+uhr39gxQYlBTm3amLiIwo766GPeGkNGYT8qYoEZHjln+JQJPSiIgcJi8TgR4mExE5JO8SQW//oMpLiIikyLtE0K2J60VEDpN3iaCnT11DIiKpIk0EZnaJmb1iZuvN7IsjvH+7ma0zszVm9h9mNjfKeCDoGtJgsYjIIZElgnC+4/uB9wELgGvNbMGwzZ4Hmt39TOBx4K+jimdIT7+6hkREUkXZIjgXWO/u7e7eBzwKXJG6gbv/1t27w8U2oDbCeAAlAhGR4aJMBLOBLSnLHeG6o/kE8OuR3jCzm8xspZmt3LVr1wkF1dOX0DSVIiIpxsVgsZldDzQD9470vrs/6O7N7t5cU1NzQsfqVYtAROQw6Uxe/2ZtBeakLNeG6w5jZu8C7gQudPeDEcaDu6trSERkmChbBCuAeWbWYGYlwDXAstQNzOxs4DvA5e6+M8JYAOgfdAYTrruGRERSRJYI3H0AuAV4EngZ+Im7rzWzu83s8nCze4GpwE/NbLWZLTvK7saE5isWETlSlF1DuPsTwBPD1n055fW7ojz+cL2anUxE5AjjYrA4U4ZmJ1OtIRGRQ/IrEahrSETkCHmVCLqH5itWi0BEJCmvEoHGCEREjpRXiWBojECJQETkkPxKBEMtgpK8Om0RkVHl1RVRg8UiIkfKq0SgMQIRkSPlVSLo0V1DIiJHyK9EMNQ1VKREICIyJO8SQWlRAQUFlu1QRETGjbxKBL19gyovISIyTF4lAs1FICJypDxLBJqmUkRkuPxKBH0DahGIiAyTX4lAXUMiIkfIr0TQN6hnCEREhsmvRNCfUHkJEZFh8ioR9KprSETkCHmVCHr6lAhERIbLr0TQrzECEZHh8i4RaIxARORweZMIBhNO30BCXUMiIsPkTSIYmotAtYZERA6XN4kgWYJaiUBE5DD5kwg0cb2IyIjyJxFomkoRkRHlTyJITlOZN6csIpKWvLkqJscI1CIQETlM3iUCdQ2JiBwubxJBb7JrSIlARCRVpInAzC4xs1fMbL2ZfXGE90vN7LHw/WfNrD6qWNQiEBEZWWSJwMwKgfuB9wELgGvNbMGwzT4BdLr7W4C/Bb4eVTyv7NgPwB9ePxDVIUREclKULYJzgfXu3u7ufcCjwBXDtrkC+EH4+nHgYjOzsQ5k1eZOvv8/GwG45UfPsWpz51gfQkQkZ0WZCGYDW1KWO8J1I27j7gNAFxAbviMzu8nMVprZyl27dh13IG3tcQYTDkD/YIK29vhx70NEZKLKicFid3/Q3Zvdvbmmpua4P7+kMUZpcQGFBsVFBSxpPCLXiIjkraII970VmJOyXBuuG2mbDjMrAiqBMf+6vmjuNB65cQlt7XGWNMZYNHfaWB9CRCRnRZkIVgDzzKyB4IJ/DfDRYdssA24AWoGrgf90d48imEVzpykBiIiMILJE4O4DZnYL8CRQCDzk7mvN7G5gpbsvA74P/KOZrQf2ECQLERHJoChbBLj7E8ATw9Z9OeV1L/ChKGMQEZHR5cRgsYiIREeJQEQkzykRiIjkOSUCEZE8ZxHdrRkZM9sFbH6TH68Gdo9hOL081n4AAAfUSURBVLlA55wfdM754UTOea67j/hEbs4lghNhZivdvTnbcWSSzjk/6JzzQ1TnrK4hEZE8p0QgIpLn8i0RPJjtALJA55wfdM75IZJzzqsxAhEROVK+tQhERGQYJQIRkTw3IROBmV1iZq+Y2Xoz++II75ea2WPh+8+aWX3moxxbaZzz7Wa2zszWmNl/mNncbMQ5lo51zinbXWVmbmY5f6thOudsZh8Of9drzexHmY5xrKXxb7vOzH5rZs+H/74vzUacY8XMHjKznWb20lHeNzO7L/z7WGNmC0/4oO4+oX4ISl5vABqBEuAFYMGwbT4FPBC+vgZ4LNtxZ+CcLwImh68/mQ/nHG5XDjwNtAHN2Y47A7/necDzwLRw+aRsx52Bc34Q+GT4egGwKdtxn+A5vx1YCLx0lPcvBX4NGLAEePZEjzkRWwTnAuvdvd3d+4BHgSuGbXMF8IPw9ePAxWZmGYxxrB3znN39t+7eHS62EcwYl8vS+T0DfBX4OtCbyeAiks45/wlwv7t3Arj7zgzHONbSOWcHKsLXlcC2DMY35tz9aYL5WY7mCuAfPNAGVJnZzBM55kRMBLOBLSnLHeG6Ebdx9wGgC8jliYzTOedUnyD4RpHLjnnOYZN5jrv/KpOBRSid3/MpwClm9jszazOzSzIWXTTSOee7gOvNrINg/pNbMxNa1hzv//djinRiGhl/zOx6oBm4MNuxRMnMCoBvAh/LciiZVkTQPfQOglbf02Z2hrvvzWpU0boWeNjdv2FmLQSzHp7u7olsB5YrJmKLYCswJ2W5Nlw34jZmVkTQnIxnJLpopHPOmNm7gDuBy939YIZii8qxzrkcOB14ysw2EfSlLsvxAeN0fs8dwDJ373f3jcCrBIkhV6Vzzp8AfgLg7q1AGUFxtokqrf/vx2MiJoIVwDwzazCzEoLB4GXDtlkG3BC+vhr4Tw9HYXLUMc/ZzM4GvkOQBHK93xiOcc7u3uXu1e5e7+71BOMil7v7yuyEOybS+bf9c4LWAGZWTdBV1J7JIMdYOuf8GnAxgJnNJ0gEuzIaZWYtA/4ovHtoCdDl7ttPZIcTrmvI3QfM7BbgSYI7Dh5y97Vmdjew0t2XAd8naD6uJxiUuSZ7EZ+4NM/5XmAq8NNwXPw1d788a0GfoDTPeUJJ85yfBN5jZuuAQeAL7p6zrd00z/nzwHfN7HMEA8cfy+Uvdmb2Y4JkXh2Oe3wFKAZw9wcIxkEuBdYD3cDHT/iYOfz3JSIiY2Aidg2JiMhxUCIQEclzSgQiInlOiUBEJM8pEYiI5DklAhmXzGzQzFan/NSPsu2BMTjew2a2MTzWc+ETqse7j++Z2YLw9Z8Pe++ZE40x3M/Q38tLZvZLM6s6xvZn5Xo1Tomebh+VccnMDrj71LHedpR9PAz8i7s/bmbvAf7G3c88gf2dcEzH2q+Z/QB41d2/Nsr2HyOounrLWMciE4daBJITzGxqOI/Cc2b2opkdUWnUzGaa2dMp35gvCNe/x8xaw8/+1MyOdYF+GnhL+Nnbw329ZGafDddNMbNfmdkL4fqPhOufMrNmM7sHmBTG8Uj43oHwz0fN7P0pMT9sZlebWaGZ3WtmK8Ia83+axl9LK2GxMTM7NzzH583sGTN7a/gk7t3AR8JYPhLG/pCZLQ+3Haliq+SbbNfe1o9+RvoheCp2dfjzM4Kn4CvC96oJnqocatEeCP/8PHBn+LqQoN5QNcGFfUq4/v8AXx7heA8DV4evPwQ8CywCXgSmEDyVvRY4G7gK+G7KZyvDP58inPNgKKaUbYZi/CDwg/B1CUEVyUnATcCXwvWlwEqgYYQ4D6Sc30+BS8LlCqAofP0u4J/C1x8D/i7l838JXB++riKoRTQl279v/WT3Z8KVmJAJo8fdzxpaMLNi4C/N7O1AguCb8AxgR8pnVgAPhdv+3N1Xm9mFBJOV/C4srVFC8E16JPea2ZcI6tR8gqB+zc/c/Y0whn8GLgB+A3zDzL5O0J3038dxXr8GvmVmpcAlwNPu3hN2R51pZleH21USFIvbOOzzk8xsdXj+LwP/lrL9D8xsHkGZheKjHP89wOVmdke4XAbUhfuSPKVEILniOqAGWOTu/RZUFC1L3cDdnw4TxfuBh83sm0An8G/ufm0ax/iCuz8+tGBmF4+0kbu/asFcB5cCf2Fm/+Hud6dzEu7ea2ZPAe8FPkIw0QoEs03d6u5PHmMXPe5+lplNJqi/82ngPoIJeH7r7h8MB9afOsrnDbjK3V9JJ17JDxojkFxRCewMk8BFwBFzLlswD/Pr7v5d4HsE0/21AeeZ2VCf/xQzOyXNY/43cKWZTTazKQTdOv9tZrOAbnf/IUExv5HmjO0PWyYjeYygUNhQ6wKCi/onhz5jZqeExxyRB7PN3QZ83g6VUh8qRfyxlE33E3SRDXkSuNXC5pEFVWklzykRSK54BGg2sxeBPwJ+P8I27wBeMLPnCb5tf8vddxFcGH9sZmsIuoVOTeeA7v4cwdjBcoIxg++5+/PAGcDysIvmK8BfjPDxB4E1Q4PFw/wrwcRA/+7B9IsQJK51wHMWTFr+HY7RYg9jWUMwMctfA38Vnnvq534LLBgaLCZoORSHsa0NlyXP6fZREZE8pxaBiEieUyIQEclzSgQiInlOiUBEJM8pEYiI5DklAhGRPKdEICKS5/4/C+f6oLwl0h4AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Q1kXebssCRR",
        "colab_type": "text"
      },
      "source": [
        "По сравнению с бейзлайном на всех данных precision метки 1 снизился до 0.81. Все остальные метрики не изминились."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zD0wGAwR8Ghj",
        "colab_type": "text"
      },
      "source": [
        "### Top 1000 features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sd-Js2db6_0-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "top1000_features = top_features[:1000]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fo9TKkaMWl4t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train_eli5 = X_train[:,top1000_features]\n",
        "X_test_eli5 = X_test[:,top1000_features]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ImLbARvbWocs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "eli5_model = SGDClassifier().fit(X_train_eli5, y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kGe-IELmXQlw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_pred = eli5_model.predict(X_test_eli5)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vc4VYCg-XuVQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "769db11e-138f-425d-c5b9-0eec6898f333"
      },
      "source": [
        "print(classification_report(y_test, y_pred))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.99      0.97     66765\n",
            "           1       0.78      0.49      0.60      7007\n",
            "\n",
            "    accuracy                           0.94     73772\n",
            "   macro avg       0.86      0.74      0.79     73772\n",
            "weighted avg       0.93      0.94      0.93     73772\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "updJ6B0cSuBS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "outputId": "ef69d417-e333-4884-da9d-1c16254139b8"
      },
      "source": [
        "y_true = y_test\n",
        "y_predicted = y_pred \n",
        "fpr, tpr, _ = roc_curve(y_true, y_predicted)\n",
        "pyplot.plot(fpr, tpr, marker='.', label='Test')\n",
        "pyplot.xlabel('False Positive Rate')\n",
        "pyplot.ylabel('True Positive Rate')\n",
        "pyplot.legend()\n",
        "pyplot.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxV9Z3/8deHfRUUUCI7ChUkqUvqWutuIeAyo3WptrWjtdaqtS5TO/bXOnYZO7adR+34q6WWH7a11epMK5ooXUaG1oICLglgtYgigSAxIkICZPv8/jjnhpsQbm7MPffm3vN+Ph555J7l3vM5RD+fe77fc75fc3dERCS++uQ6ABERyS0VAhGRmFMhEBGJORUCEZGYUyEQEYm5frkOoLtGjx7tkydPznUYIiJ5ZdWqVe+4+5jOtuVdIZg8eTIrV67MdRgiInnFzDbsb5uahkREYk6FQEQk5lQIRERiLu/6CDrT1NREdXU1u3fvznUoGTdo0CDGjx9P//79cx2KiBSogigE1dXVDB8+nMmTJ2NmuQ4nY9yduro6qqurmTJlSq7DEZECFVnTkJktMLOtZrZ6P9vNzO41s3VmVmlmx3zQY+3evZtRo0YVVBEAMDNGjRpVkFc6ItJ7RNlHsBCYnWL7HGBa+HMN8OOeHKzQikBCoZ6XiHTPqg3buO+ZdazasC3jnx1Z05C7LzWzySl2OR/4uQfjYC83s5FmVuTuNVHFJCKSj3734iZuffRlWt0Z0K8PD119AsdOOjBjn5/LPoJxwMak5epw3T6FwMyuIbhqYOLEiVkJrjvq6uo488wzAdiyZQt9+/ZlzJjgAb7nn3+eAQMGpHz/kiVLGDBgACeddFLksYpIfnjt7R2UV9ZQUVXD37fubFvf1NzK8vV1BVMI0ubu84H5AKWlpb1uJp1Ro0bx0ksvAXDnnXcybNgwbr311rTfv2TJEoYNG6ZCIBJzf397B+VVNZRXBsnfDD4y+SCu/ugUfrF8A80trfTv14cTpo7K6HFzWQg2AROSlseH67Ji1YZtLF9fxwlTR2W0srZ9/qpV3HzzzezcuZPRo0ezcOFCioqKuPfee7n//vvp168fM2fO5O677+b++++nb9++/PKXv+RHP/oRp5xySsbjEZHeKZH8K6pqeO3tvcn/X887kjmzxnLwAYMAmFNcFFnOymUhWARcb2YPA8cD2zPRP/CvT6xh7eb3U+6zY3cTf9uyg1aHPgZHjB3O8EH7v09/5qEH8I1zj0w7Bnfnhhtu4PHHH2fMmDE88sgj3HHHHSxYsIC7776bN954g4EDB/Lee+8xcuRIrr322m5fRYhI/lq3dQdPVnZI/pOC5D971lgOCZN/smMnHRjJl1aIsBCY2a+B04DRZlYNfAPoD+Du9wMVQBmwDmgAPhtVLB29v7uZ1rCBqdWD5VSFoLv27NnD6tWrOfvsswFoaWmhqKgIgJKSEi6//HIuuOACLrjggowdU0R6t3Vbd1BeuYXyqs3tkv+d585kTnFRp8k/W6K8a+iyLrY78MVMHzedb+6rNmzj8geW09QctLf98NKjM1pp3Z0jjzySZcuW7bOtvLycpUuX8sQTT/Dtb3+bqqqqjB1XRHqXRPKvqKrh1bd39KrknywvOosz7dhJB/LQ1SdE1t42cOBAamtrWbZsGSeeeCJNTU289tprzJgxg40bN3L66afz0Y9+lIcffpidO3cyfPhw3n8/dXOWiOSHdVt3tt3tk0j+pZMO7HXJP1ksCwFE297Wp08fHnvsMW688Ua2b99Oc3MzN910E9OnT+eKK65g+/btuDs33ngjI0eO5Nxzz+Wiiy7i8ccfV2exSB5at3UnFeHdPsnJ/xvnzmTOrCLGjuh9yT+ZBS00+aO0tNQ7TkzzyiuvMGPGjBxFFL1CPz+RfJRI/hVVNfxtyw4APjL5QMqKi3pl8jezVe5e2tm22F4RiIh01+u1e5t9Esk/n775748KgYhICq/X7qSisobyDsn/6/NmMqd4LEUjBuc4wp4rmELg7gU5QFu+Nd2JFIL14Tf/5OR/bIEl/2QFUQgGDRpEXV1dwQ1FnZiPYNCg/LzcFMkn62uDNv8nK9sn//8zbyZlBZj8kxVEIRg/fjzV1dXU1tbmOpSMS8xQJiKZl0j+5VVbeKUmuIU7kfznzBrLoSMLN/knK4hC0L9/f83gJSJp6Sz5HzNxZOySf7KCKAQiIqm88U59W7NPcvL/2twZlBUXxTL5J1MhEJGClEj+5ZU1rFXyT0mFQEQKRmfJ/2gl/y6pEIhIXnvznfq2yVw6Jv85xUWMU/LvkgqBiOSdRPKvqKphzWYl/55SIRCRvLChbu83/0TyP2qCkn8mqBCISK+VSP4VVTWs3rQ3+d9RNoM5xWMZf+CQHEdYGFQIRKRXUfLPPhUCEcm5t+oagmafqs1tyf/DSv5Zo0IgIjmRSP4VVTVUbdoOBMn/X8qOYM6sIiYcpOSfLSoEIpI1Sv69kwqBiERq47sNbXf7tCX/8SOU/HsRFQIRybhE8q+oqqGyem/y/+qcIygrVvLvbVQIRCQjNr7bEI7quTf5lyj55wUVAhH5wBLJv6KqhpeTkv/tc45grpJ/3lAhEJFuSZX8y2YVMXGUkn++USEQkS5tfLeBp1YHHb6J5F88Tsm/UKgQiEinqrc1tM3k9fLG94Ag+X9ldtDso+RfOFQIRKSNkn88qRCIxFz1tgaeqtrCk1U1bcl/1rgD+MrsIygrHsukUUNzHKFETYVAJIYSyb+8qoaXkpL/P8/+EHOLi5T8Y0aFQCQmNr23i6fCCdyV/CVZpIXAzGYDPwT6Ag+4+90dtk8EHgRGhvvc7u4VUcYkEieJ5F9eVcOLbwXJ/8hDg+RfNquIyaOV/CXCQmBmfYH7gLOBamCFmS1y97VJu30N+I27/9jMZgIVwOSoYhKJg/0l/9s+HnzzV/KXjqK8IjgOWOfu6wHM7GHgfCC5EDhwQPh6BLA5wnhECtbm93a1De+g5C/dFWUhGAdsTFquBo7vsM+dwO/N7AZgKHBWZx9kZtcA1wBMnDgx44GK5KNE8q+oquGFMPnPLFLyl+7LdWfxZcBCd/++mZ0I/MLMZrl7a/JO7j4fmA9QWlrqOYhTpFdIlfzLiouYouQvH0CUhWATMCFpeXy4LtlVwGwAd19mZoOA0cDWCOMSySs123dRUbWF8srNbcl/hpK/ZFCUhWAFMM3MphAUgEuBT3bY5y3gTGChmc0ABgG1EcYkkhcSyb+iqoZVG7YBQfK/9ZzplBUXMXXMsBxHKIUkskLg7s1mdj2wmODW0AXuvsbM7gJWuvsi4Bbgp2b2ZYKO4yvdXU0/Ektbtu9u6/BV8pdssnzLu6Wlpb5y5cpchyGSEYnkX1FVw8ow+R8xdjjzSoqU/CWjzGyVu5d2ti3XncUisbO/5H/L2dMpKyniMCV/yTIVApEs2LJ9d9t4/kr+0tuoEIhEJJH8K6pqWPGmkr/0XioEIhn09vvtm33cg+R/89lBh+/hByv5S++jQiDSQ2+/v7ttbJ/k5P/ls5T8JT+oEIh8AInkX1G1hRUb3sUdPnSIkr/kJxUCkTTtL/nfdOZ05paM5fCDh+c6RJEPRIVAJIWt7+/mqdVbKK+sUfKXgqVCINJBW/KvqmHFm0Hyn37IMCV/KVgqBCLA1h27eXr1Fp6sbJ/8v3TmNOYWFzHtECV/KVxpFwIzG+LuDVEGI5JNieRfXlnD82Hyn3awkr/ET5eFwMxOAh4AhgETzezDwOfd/bqogxPJtP0l/xvPmMbckiKmK/lLDKVzRfAfwMeBRQDu/rKZfSzSqEQyaOuO3SwOm30Syf9wJX+RNmk1Dbn7RjNLXtUSTTgimZFI/uVVNTz3hpK/SCrpFIKNYfOQm1l/4EvAK9GGJdJ9tTv28PTq4Anf5994l9Yw+d9wxjTmKfmL7Fc6heBa4IcEk9FvAn4PqH9AeoXaHXt4ek0wjWMi+R82ZijXnxF0+E4/ZBgdrmZFpIN0CsGH3P3y5BVmdjLwbDQhiaSWSP4VlTU890adkr9ID6VTCH4EHJPGOpHIvLNzT9vdPu2S/+mHM7fkUCV/kR7YbyEwsxOBk4AxZnZz0qYDCOYgFolUIvlXVNWwfH2Q/KeGyb+spIgPHTJcyV8kA1JdEQwgeHagH5Dcy/Y+cFGUQUl8KfmLZN9+C4G7/y/wv2a20N03ZDEmiZm6nYkO3/bJ/4unH85cJX+RyKXTR9BgZvcARwKDEivd/YzIopKCl0j+FVU1LHs9TP6jg+RfVlzEEWOV/EWyJZ1C8BDwCDCP4FbSzwC1UQYlhUnJX6R3SqcQjHL3n5nZl5Kai1ZEHZgUhrqde1i85m3KqzazfP27tLQ6U0cP5brTgmYfJX+R3EunEDSFv2vMbC6wGTgoupAk3yWSf0VVDcvW19HS6kwZPZQvnHoYZcVFzChS8hfpTdIpBN8ysxHALQTPDxwA3BRpVJJ33q1vZHHY4Zuc/K89dSpziw9V8hfpxbosBO7+ZPhyO3A6tD1ZLDGXSP4VVTX89fUg+U8eNYRrT51KWXERM4sOUPIXyQOpHijrC1xMMMbQ0+6+2szmAf8CDAaOzk6I0pso+YsUnlRXBD8DJgDPA/ea2WagFLjd3X+XjeCkd9iWaPbpkPw//7GpzC1R8hfJd6kKQSlQ4u6tZjYI2AIc5u512QlNcilV8i8rLuLIQ5X8RQpFqkLQ6O6tAO6+28zWd7cImNlsgiGs+wIPuPvdnexzMXAn4MDL7v7J7hxDMqez5D9JyV+k4KUqBEeYWWX42oDDwmUD3N1LUn1w2MdwH3A2UA2sMLNF7r42aZ9pwFeBk919m5kd3INzkQ9gW30jv18bTOOYnPyv+dhU5ir5i8RCqkIwo4effRywzt3XA5jZw8D5wNqkfT4H3Ofu2wDcfWsPjylpSCT/8qot/HXdOzQr+YvEWqpB53o60Nw4YGPScjVwfId9pgOY2bMEzUd3uvvTHT/IzK4BrgGYOHFiD8OKp/caGvn9mrd5sqqmLflPPGgIn1PyF4m9tCavj/j404DTgPHAUjMrdvf3kndy9/nAfIDS0lLPdpD5KpH8y6tqeDYp+V99ylTmlSj5i0ggykKwieD204Tx4bpk1cBz7t4EvGFmrxEUBo1l9AGlSv5zi4uYNU7JX0TaS6sQmNlgYKK7v9qNz14BTDOzKQQF4FKg4x1BvwMuA/6fmY0maCpa341jCLC9oYnFa4PhHRLJf8JBg5X8RSQtXRYCMzsX+B7BjGVTzOwo4C53Py/V+9y92cyuBxYTtP8vcPc1ZnYXsNLdF4XbzjGztUALcJueU0hPIvlXVNXwl7/vTf5XnTKFecWHKvmLSNrMPXWTu5mtAs4Alrj70eG6KncvzkJ8+ygtLfWVK1fm4tA5l5z8n133Dk0tzvgDBzO3pIi5xUUUjxuh5C8inTKzVe5e2tm2tIahdvftHRKMOmyzZHtDU3irZ/vk/08nT2FuiZK/iPRcOoVgjZl9EugbPgB2I/DXaMOKt0Tyr6iq4S8dkn9ZcREl45X8RSRz0ikENwB3AHuAXxG0638ryqDiqLPkP26kkr+IRC+dQnCEu99BUAwkg7bvauIPa9+mvHJzu+T/2ZOnMFfJX0SyJJ1C8H0zGws8Bjzi7qsjjqmgJZJ/RVUNf/57bbvkX1ZcxIeV/EUky9KZoez0sBBcDPzEzA4gKAhqHkrT9l1N/HFt8JCXkr+I9DZpPVDm7lsIJqd5Bvhn4OuonyClRPKvqKphaVLyv/KkycwtOVTJX0R6jXQeKJsBXAJcCNQBjxBMZC8dvL+7iT+sSTT7vENjS2tb8i8rLuKoCSOV/EWk10nnimABQfL/uLtvjjievPP+7rDZp7J98v/MSZOU/EUkL6TTR3BiNgLJJ4nkX1FVw9LXguR/6IhBfPrEScwtUfIXkfyy30JgZr9x94vNrIr2TxKnNUNZoUmV/MtKijhayV9E8lSqK4Ivhb/nZSOQ3mjH7ib++ErQ7NNZ8j9q/Ej69FHyF5H8lmqGsprw5XXu/pXkbWb2XeAr+74r/+1N/ltY+vdaGpuD5P+pRLOPkr+IFJh0OovPZt+kP6eTdXln1YZtLF9fR8n4EdTtbOTJypq25F80YhCfOiHo8D16gpK/iBSuVH0EXwCuA6aaWWXSpuHAs1EHFrVVG7Zx6fxlNLXs7f5Q8heROEp1RfAr4Cng34Dbk9bvcPd3I40qC55dV9tWBAz45PET+eb5s5T8RSR2UhUCd/c3zeyLHTeY2UH5XgyKx48EgiIwsH8f/vGY8SoCIhJLXV0RzANWEdw+mpwlHZgaYVyRO3zMMABmzxrL1adM5dhJB+Y4IhGR3Eh119C88PeU7IWTPQ2NLQDMKzlURUBEYq1PVzuY2clmNjR8fYWZ/cDMJkYfWrQaGpsBGDKwb44jERHJrS4LAfBjoMHMPkww2NzrwC8ijSoLElcEQwekNQCriEjBSqcQNLu7A+cD/+nu9xHcQprXEoVgyABdEYhIvKXzdXiHmX0V+BRwipn1AfpHG1b02pqGVAhEJObSuSK4hGDi+n8KJ6gZD9wTaVRZUL8nbBoaqKYhEYm3LgtBmPwfAkaY2Txgt7v/PPLIIpa4IhisKwIRibl07hq6GHge+ATBvMXPmdlFUQcWtbY+gv4qBCISb+m0i9wBfMTdtwKY2Rjgj8BjUQYWtfrGZgb060O/vum0jomIFK50smCfRBEI1aX5vl5tV2MLQ9UsJCKS1hXB02a2GPh1uHwJUBFdSNlRv6eFIXqGQEQkrTmLbzOzfwQ+Gq6a7+6/jTas6O1qatatoyIipJ6PYBrwPeAwoAq41d03ZSuwqNXvaWGIbh0VEUnZ1r8AeBK4kGAE0h9198PNbLaZvWpm68zs9hT7XWhmbmal3T3GB9XQ2Kw+AhERUjcNDXf3n4avXzWzF7rzwWbWF7iPYKrLamCFmS1y97Ud9hsOfAl4rjuf31MNjS0Ujcj7B6RFRHosVSEYZGZHs3cegsHJy+7eVWE4Dljn7usBzOxhgvGK1nbY75vAd4Hbuhl7jzQ0qrNYRARSF4Ia4AdJy1uSlh04o4vPHgdsTFquBo5P3sHMjgEmuHu5me23EJjZNcA1ABMnZmYE7Po96iwWEYHUE9OcHuWBw8HrfgBc2dW+7j4fmA9QWlrqXeyell26IhARAaJ9MGwTMCFpeXy4LmE4MAtYYmZvAicAi7LRYezu1Dc2M1ST0oiIRFoIVgDTzGyKmQ0ALgUWJTa6+3Z3H+3uk919MrAcOM/dV0YYEwB7mltpdQ04JyICERYCd28GrgcWA68Av3H3NWZ2l5mdF9Vx06HZyURE9uoyE5qZAZcDU939rnC+4rHu/nxX73X3CjoMR+HuX9/PvqelFXEG1O/RpDQiIgnpXBH8X+BE4LJweQfB8wF5a1dTYppKXRGIiKSTCY9392PM7EUAd98WtvnnrbYrAnUWi4ikdUXQFD4l7NA2H0FrpFFFTJPSiIjslU4huBf4LXCwmX0b+AvwnUijilhbZ7EGnRMRSWsY6ofMbBVwJsHwEhe4+yuRRxahxHzF6iwWEUnvrqGJQAPwRPI6d38rysCi1NY0pM5iEZG0OovLCfoHDBgETAFeBY6MMK5IqbNYRGSvdJqGipOXw4HirossoixQZ7GIyF7dfrI4HH76+C537MUaGlsY0K8P/fpGOcKGiEh+SKeP4OakxT7AMcDmyCLKAs1OJiKyVzp9BMOTXjcT9Bn8VzThZEf9Hg1BLSKSkDIbhg+SDXf3W7MUT1bsatKkNCIiCfttJDezfu7eApycxXiyon5PC0P0MJmICJD6iuB5gv6Al8xsEfAoUJ/Y6O7/HXFskWlobNYdQyIioXS+Fg8C6gjmKE48T+BAHheCFopG9M91GCIivUKqQnBweMfQavYWgISMzBucKw2ar1hEpE2qbNgXGEb7ApCQ54VAncUiIgmpCkGNu9+VtUiyqEG3j4qItEn1aG1nVwJ5z92pb2xmqMYZEhEBUheCM7MWRRbtaW6l1WGwmoZERIAUhcDd381mINnSNimNmoZERIAPMOhcvksMQa0rAhGRQOwKwa4mXRGIiCSLXSHQpDQiIu3FrhDs0qQ0IiLtxK4Q1Cc6izXonIgIEMNC0NAYNg2ps1hEBIhlIQibhtRZLCICxLAQqLNYRKS92BWCBnUWi4i0E2khMLPZZvaqma0zs9s72X6zma01s0oz+5OZTYoyHggKwYB+fejXN3Y1UESkU5Flw3C+4/uAOcBM4DIzm9lhtxeBUncvAR4D/j2qeBIaGpsZqo5iEZE2UX4tPg5Y5+7r3b0ReBg4P3kHd3/G3RvCxeXA+AjjATQpjYhIR1EWgnHAxqTl6nDd/lwFPNXZBjO7xsxWmtnK2traHgWlSWlERNrrFQ3lZnYFUArc09l2d5/v7qXuXjpmzJgeHat+TwtD9DCZiEibKDPiJmBC0vL4cF07ZnYWcAdwqrvviTAeIBhiQncMiYjsFeUVwQpgmplNMbMBwKXAouQdzOxo4CfAee6+NcJY2mh2MhGR9iIrBO7eDFwPLAZeAX7j7mvM7C4zOy/c7R5gGPComb1kZov283EZ09DYwmB1FouItIk0I7p7BVDRYd3Xk16fFeXxO6PbR0VE2usVncXZ1LBHt4+KiCSLVSFwdxqaWnT7qIhIklgVgj3NrbS0ugacExFJEqtCkBhwTvMVi4jsFbNCEAxBPVhNQyIibWJWCHRFICLSUawKQdukNLoiEBFpE6tCsKttmkoVAhGRhFgVgvpE05AGnRMRaROrQqDOYhGRfcWsEKizWESko1gVgrbOYj1QJiLSJlaFoK2zWPMRiIi0iVUhqG9sYUC/PvTrG6vTFhFJKVYZUfMVi4jsK2aFoEUdxSIiHcSsEOiKQESko5gVAs1FICLSUbwKgWYnExHZR6wKQX1jM0P1DIGISDuxKgS7GlsYrCsCEZF2YlUI6hubGao+AhGRdmJVCBr2tGjAORGRDmJTCNydhiY9RyAi0lFsCsGe5lZaWl0DzomIdBCbQqAB50REOhebQlDfmBiCWk1DIiLJYlMINCmNiEjnYlcINMSEiEh78SkEidnJVAhERNqJTSGob7siUNOQiEiySAuBmc02s1fNbJ2Z3d7J9oFm9ki4/TkzmxxVLA2Nmq9YRKQzkRUCM+sL3AfMAWYCl5nZzA67XQVsc/fDgf8AvhtVPK/UvA/A+q31UR1CRCQvRXlFcBywzt3Xu3sj8DBwfod9zgceDF8/BpxpZpbpQFZt2MYDf34DgOt//QKrNmzL9CFERPJWlIVgHLAxabk6XNfpPu7eDGwHRnX8IDO7xsxWmtnK2trabgeyfH0dLa0OQHNLK8vX13X7M0REClVedBa7+3x3L3X30jFjxnT7/SdMHcXA/n3oa9C/Xx9OmLpPrRERia0ob6HZBExIWh4frutsn2oz6weMADL+df3YSQfy0NUnsHx9HSdMHcWxkw7M9CFERPJWlIVgBTDNzKYQJPxLgU922GcR8BlgGXAR8D/u7lEEc+ykA1UAREQ6EVkhcPdmM7seWAz0BRa4+xozuwtY6e6LgJ8BvzCzdcC7BMVCRESyKNKnq9y9AqjosO7rSa93A5+IMgYREUktLzqLRUQkOioEIiIxp0IgIhJzKgQiIjFnEd2tGRkzqwU2fMC3jwbeyWA4+UDnHA8653joyTlPcvdOn8jNu0LQE2a20t1Lcx1HNumc40HnHA9RnbOahkREYk6FQEQk5uJWCObnOoAc0DnHg845HiI551j1EYiIyL7idkUgIiIdqBCIiMRcQRYCM5ttZq+a2Tozu72T7QPN7JFw+3NmNjn7UWZWGud8s5mtNbNKM/uTmU3KRZyZ1NU5J+13oZm5meX9rYbpnLOZXRz+rdeY2a+yHWOmpfHf9kQze8bMXgz/+y7LRZyZYmYLzGyrma3ez3Yzs3vDf49KMzumxwd194L6IRjy+nVgKjAAeBmY2WGf64D7w9eXAo/kOu4snPPpwJDw9RficM7hfsOBpcByoDTXcWfh7zwNeBE4MFw+ONdxZ+Gc5wNfCF/PBN7Mddw9POePAccAq/ezvQx4CjDgBOC5nh6zEK8IjgPWuft6d28EHgbO77DP+cCD4evHgDPNzLIYY6Z1ec7u/oy7N4SLywlmjMtn6fydAb4JfBfYnc3gIpLOOX8OuM/dtwG4+9Ysx5hp6ZyzAweEr0cAm7MYX8a5+1KC+Vn253zg5x5YDow0s6KeHLMQC8E4YGPScnW4rtN93L0Z2A7k80TG6ZxzsqsIvlHksy7PObxknuDu5dkMLELp/J2nA9PN7FkzW25ms7MWXTTSOec7gSvMrJpg/pMbshNaznT3//cuRToxjfQ+ZnYFUAqcmutYomRmfYAfAFfmOJRs60fQPHQawVXfUjMrdvf3chpVtC4DFrr7983sRIJZD2e5e2uuA8sXhXhFsAmYkLQ8PlzX6T5m1o/gcrIuK9FFI51zxszOAu4AznP3PVmKLSpdnfNwYBawxMzeJGhLXZTnHcbp/J2rgUXu3uTubwCvERSGfJXOOV8F/AbA3ZcBgwgGZytUaf3/3h2FWAhWANPMbIqZDSDoDF7UYZ9FwGfC1xcB/+NhL0ye6vKczexo4CcERSDf242hi3N29+3uPtrdJ7v7ZIJ+kfPcfWVuws2IdP7b/h3B1QBmNpqgqWh9NoPMsHTO+S3gTAAzm0FQCGqzGmV2LQI+Hd49dAKw3d1revKBBdc05O7NZnY9sJjgjoMF7r7GzO4CVrr7IuBnBJeP6wg6ZS7NXcQ9l+Y53wMMAx4N+8XfcvfzchZ0D6V5zgUlzXNeDJxjZmuBFuA2d8/bq900z/kW4Kdm9mWCjuMr8/mLnZn9mqCYjw77Pb4B9Adw9/sJ+kHKgHVAA/DZHh8zj/+9REQkAwqxaUhERLpBhUBEJOZUCEREYk6FQEQk5lQIRERiToVAeiUzazGzl5J+JqfYd2cGjrfQzN4Ij/VC+IRqdz/jATObGb7+lw7b/trTGMPPSfy7rDazJ8xsZBf7H5Xvo3FK9HT7qPRKZrbT3Ydlet8Un7EQeNLdHzOzc4DvuXtJDz6vx+WR5CQAAAOkSURBVDF19blm9iDwmrt/O8X+VxKMunp9pmORwqErAskLZjYsnEfhBTOrMrN9Rho1syIzW5r0jfmUcP05ZrYsfO+jZtZVgl4KHB6+9+bws1ab2U3huqFmVm5mL4frLwnXLzGzUjO7GxgcxvFQuG1n+PthM5ubFPNCM7vIzPqa2T1mtiIcY/7zafyzLCMcbMzMjgvP8UUz+6uZfSh8Evcu4JIwlkvC2BeY2fPhvp2N2Cpxk+uxt/Wjn85+CJ6KfSn8+S3BU/AHhNtGEzxVmbii3Rn+vgW4I3zdl2C8odEEiX1ouP4rwNc7Od5C4KLw9SeA54BjgSpgKMFT2WuAo4ELgZ8mvXdE+HsJ4ZwHiZiS9knE+A/Ag+HrAQSjSA4GrgG+Fq4fCKwEpnQS586k83sUmB0uHwD0C1+fBfxX+PpK4D+T3v8d4Irw9UiCsYiG5vrvrZ/c/hTcEBNSMHa5+1GJBTPrD3zHzD4GtBJ8Ez4E2JL0nhXAgnDf37n7S2Z2KsFkJc+GQ2sMIPgm3Zl7zOxrBOPUXEUwfs1v3b0+jOG/gVOAp4Hvm9l3CZqT/tyN83oK+KGZDQRmA0vdfVfYHFViZheF+40gGCzujQ7vH2xmL4Xn/wrwh6T9HzSzaQTDLPTfz/HPAc4zs1vD5UHAxPCzJKZUCCRfXA6MAY519yYLRhQdlLyDuy8NC8VcYKGZ/QDYBvzB3S9L4xi3uftjiQUzO7Ozndz9NQvmOigDvmVmf3L3u9I5CXffbWZLgI8DlxBMtALBbFM3uPviLj5il7sfZWZDCMbf+SJwL8EEPM+4+z+EHetL9vN+Ay5091fTiVfiQX0Eki9GAFvDInA6sM+cyxbMw/y2u/8UeIBgur/lwMlmlmjzH2pm09M85p+BC8xsiJkNJWjW+bOZHQo0uPsvCQbz62zO2KbwyqQzjxAMFJa4uoAgqX8h8R4zmx4es1MezDZ3I3CL7R1KPTEU8ZVJu+4gaCJLWAzcYOHlkQWj0krMqRBIvngIKDWzKuDTwN862ec04GUze5Hg2/YP3b2WIDH+2swqCZqFjkjngO7+AkHfwfMEfQYPuPuLQDHwfNhE8w3gW528fT5Qmegs7uD3BBMD/dGD6RchKFxrgRcsmLT8J3RxxR7GUkkwMcu/A/8Wnnvy+54BZiY6iwmuHPqHsa0JlyXmdPuoiEjM6YpARCTmVAhERGJOhUBEJOZUCEREYk6FQEQk5lQIRERiToVARCTm/j+i5eR2pE55/AAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d5WhdAIO-eul",
        "colab_type": "text"
      },
      "source": [
        "По сравнению с бейзлайном на всех данных precision метки 1 снизился до 0.70, recall меток 0 и 1 снизился до 0.98 и 0.49, f1-score снизился до 0.93."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EFj_fzeK8STC",
        "colab_type": "text"
      },
      "source": [
        "### Top 10000 features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lr673l5u8WFu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "top10000_features = top_features[:10000]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "khaRoE768Ykf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train_eli5 = X_train[:,top10000_features]\n",
        "X_test_eli5 = X_test[:,top10000_features]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G8Igm0Gx8Yvg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "eli5_model = SGDClassifier().fit(X_train_eli5, y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XGqxO9Mk8Y5U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_pred = eli5_model.predict(X_test_eli5)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KlMfwmv-8cjn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "90dacc6d-525f-4b02-bf2f-8ed35dab7972"
      },
      "source": [
        "print(classification_report(y_test, y_pred))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.99      0.97     66765\n",
            "           1       0.79      0.49      0.61      7007\n",
            "\n",
            "    accuracy                           0.94     73772\n",
            "   macro avg       0.87      0.74      0.79     73772\n",
            "weighted avg       0.93      0.94      0.93     73772\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vQXMD7O8S3Gm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "outputId": "7668bbf8-f06a-465e-b65a-d4cbfca53882"
      },
      "source": [
        "y_true = y_test\n",
        "y_predicted = y_pred \n",
        "fpr, tpr, _ = roc_curve(y_true, y_predicted)\n",
        "pyplot.plot(fpr, tpr, marker='.', label='Test')\n",
        "pyplot.xlabel('False Positive Rate')\n",
        "pyplot.ylabel('True Positive Rate')\n",
        "pyplot.legend()\n",
        "pyplot.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3xV5ZX/8c9KCIRLuIUA4Q4KQjCpl1TxLuIFCF5malutztQZO04vajvWTu3YX+s47Ywdp51X7fgba60/O61TbZ1pSw2KrZXSWlDAS8JFEEEkkECI3BPIbf3+2DvhEEI4mOxzcs75vl+vvDj7cs5eG8haZz/P3s9j7o6IiGSurGQHICIiyaVCICKS4VQIREQynAqBiEiGUyEQEclwfZIdwMkaMWKET5o0KdlhiIiklFWrVu1y94LOtqVcIZg0aRIrV65MdhgiIinFzLYcb5uahkREMpwKgYhIhlMhEBHJcCnXR9CZpqYmqqqqOHToULJD6XG5ubmMGzeOnJycZIciImkqLQpBVVUVeXl5TJo0CTNLdjg9xt2pq6ujqqqKyZMnJzscEUlTkTUNmdnjZrbTzFYfZ7uZ2UNmttHMKszsrA96rEOHDpGfn59WRQDAzMjPz0/LKx0R6T2i7CN4ApjbxfZ5wNTw5zbgP7tzsHQrAm3S9bxE5OSs2rKbh1/ayKotu3v8syNrGnL3pWY2qYtdrgX+y4NxsJeb2VAzK3T36qhiEhFJRb98fRt3//xNWt3p2yeLJz81i7MnDuuxz09mH8FYYGvMclW47phCYGa3EVw1MGHChIQEdzLq6uqYM2cOADU1NWRnZ1NQEDzA9+qrr9K3b98u379kyRL69u3L+eefH3msIpIa3t6xn/LKasorqnl754H29U3NrSzfVJc2hSBu7v4o8ChAaWlpr5tJJz8/nzfeeAOA++67j0GDBnH33XfH/f4lS5YwaNAgFQKRDLdx537KK2oor9zOhh0HMIMPTxrOpy6czI+Xb6G5pZWcPlnMmpLfo8dNZiHYBoyPWR4XrkuIVVt2s3xTHbOm5PdoZW3//FWruOuuuzhw4AAjRozgiSeeoLCwkIceeohHHnmEPn36UFRUxAMPPMAjjzxCdnY2P/nJT/je977HRRdd1OPxiEjvtHHnARaF3/zX79gfJP+Jw7nv6iLmFRcyanAuAPOKCyPLWcksBAuB283sKeBcYG9P9A/846/XsHb7vi732X+oibdq9tPqkGUwfXQeebnHv0+/aMxgvn71zLhjcHfuuOMOfvWrX1FQUMDTTz/Nvffey+OPP84DDzzA5s2b6devH3v27GHo0KF8+tOfPumrCBFJXe/UHmBRRTXlldW8VbMfgA9PGnZM8o919sRhkXxphQgLgZn9FLgUGGFmVcDXgRwAd38EWATMBzYC9cBfRRVLR/sONdMaNjC1erDcVSE4WYcPH2b16tVcccUVALS0tFBYWAhASUkJN910E9dddx3XXXddjx1TRHq3TbXBN/9nK44k/9KJw/j61UXMO72Q0UOOTf6JEuVdQzeeYLsDn+vp48bzzX3Vlt3c9NhympqD9rbv3nBmj1Zad2fmzJksW7bsmG3l5eUsXbqUX//613zzm9+ksrKyx44rIr1LW/Ivr6xhXXXQUnH2xGF8bUER84pHUzikf5IjDKREZ3FPO3viMJ781KzI2tv69etHbW0ty5Yt47zzzqOpqYkNGzYwY8YMtm7dyuzZs7nwwgt56qmnOHDgAHl5eezb13Vzloikhs27Dra3+a8Nk/9ZE4byfxYUMe/00YwZ2juSf6yMLAQQbXtbVlYWzzzzDHfeeSd79+6lubmZL3zhC0ybNo2bb76ZvXv34u7ceeedDB06lKuvvprrr7+eX/3qV+osFklB7+46SHllNYsqq1kT9lGeOWEoXy2bwfziwl6Z/GNZ0EKTOkpLS73jxDTr1q1jxowZSYooeul+fiKpaEvdwfb7/NuS/xnjh7KgpJB5xYWM7WXJ38xWuXtpZ9sy9opARORkvVdXHyT/yu2s3nYk+d87fwbzikczbtiAJEf4wagQiIh0Yev79e3f/Cu37QXgQ+OH8g/zpzPv9ELGD0/N5B8rbQqBu6flAG2p1nQnkg62vl8f3u1TTUVVmPzHDeEr86Yzvzg9kn+stCgEubm51NXVpd1Q1G3zEeTmJu/+YpFMsfX9ep5bHXzzfzNM/iVpnPxjpUUhGDduHFVVVdTW1iY7lB7XNkOZiPS8qt31PFdZw7OV1by5dQ8AxWOHcM+86cw/vZAJ+emb/GOlRSHIycnRDF4iEpdtexrah3d4I0z+p48dzJfnTqesOHOSf6y0KAQiIl3ZtqeB58I2/9ffC5L/zDGD+fu5p1FWXMjE/IFJjjC5VAhEJC1t39PAovAhr9fC5F9UOJgvXRUk/0kjMjv5x1IhEJG0Ub23gUWVNZRXbD8m+c8vLmSykn+nVAhEJKXV7D3Ufqtn23y+MwoHc/eV05hfXMiUgkFJjrD3UyEQkZRTs/dQ+62eK8PkP310Hl+8YhrzSwo5Rcn/pKgQiEhK2LHvUHuH78otu3FX8u8pKgQi0mvt3HeI51bXUF5RzYot7+MOp43K4+8uD5p9Th2p5N8TVAhEpFfZuf8Qz6+u4dmKala8GyT/aaMG8YU50ygrGc2pI/OSHWLaUSEQkaRrS/7lFdW8Gib/qSMH8fk5UykrLmTqKCX/KKkQiEhS1O4/zPOrgzb/VzYHyf/UkYO487KplJUUMk3JP2FUCEQkYWr3H+b5NTUsqqjmlc11tDqcUjCQOy6bygIl/6RRIRCRSO06cLi92act+U8pGMjts0+lrGQM00YNSqtRg1ORCoGI9Li6A8E3//KKapZvCpP/iIF8bvaplJUUctqoPCX/XkSFQER6RN2Bwyxes4Pyyu0seydI/pNHDOSzlwbJf/poJf/eSoVARD6w9w82sjj85r9sUx0trd6e/OcXFzKjUMk/FagQiMhJ2d2W/Cur+dM7QfKflD+AT18yhfnFhRQVDlbyTzEqBCJyQrsPNvLC2uAhr7bkPzF/AH97cZD8Z45R8k9lKgQi0qm25F9eWcOfNu6iudWZMHwAt108hTIl/7SiQiAi7fbUN/LCmh2UV1bzcpj8xw/vz6cumsKCEiX/dKVCIJLh9tY3sXhtDYsqq/nj20HyHzesP7deNJkFxWM4faySf7pTIRDJQHvrm8Jmn+Cbf1NLmPwvnExZSSHFY4co+WcQFQKRDLG3oYnfrN1BecV2/hgm/7FD+/NXF0ymrLiQknFK/pkq0kJgZnOB7wLZwGPu/kCH7ROAHwFDw33ucfdFUcYkkkn2NjTx27VBm/8f3q49KvnPLy7kQ0r+QoSFwMyygYeBK4AqYIWZLXT3tTG7fRX4mbv/p5kVAYuASVHFJJIJ9h0Kk39FNX94exeNLa2MGZLLLedPYn5xIWeMH6rkL0eJ8orgHGCju28CMLOngGuB2ELgwODw9RBge4TxiKSt/Yea+O26IPkv3XAk+f/leROZX1LImUr+0oUoC8FYYGvMchVwbod97gNeMLM7gIHA5Z19kJndBtwGMGHChB4PVCQV7T/UxIvrdvJsRTVLN9TS2NJK4ZBc/uK8icwvDpJ/VpaSv5xYsjuLbwSecPdvm9l5wI/N7HR3b43dyd0fBR4FKC0t9STEKdIrtCX/8spqfr+hlsbmVkYPzuXmWRMpK1Hylw8mykKwDRgfszwuXBfrVmAugLsvM7NcYASwM8K4RFLKgcPNvBg2+ywJk/+owf246dwJLCgp5Mzxw5T8pVuiLAQrgKlmNpmgANwAfKLDPu8Bc4AnzGwGkAvURhiTSEpoS/6LKqt5af2R5P+Jc4Lkf9YEJX/pOZEVAndvNrPbgcUEt4Y+7u5rzOx+YKW7LwS+CPzAzP6OoOP4FndX049kpIOHm3nxrZ2UV2xnyfpaDje3MjIvSP5lJYWcreQvEYm0jyB8JmBRh3Vfi3m9FrggyhhEerODh5v53Vs7Ka+o5qX1O9uT/43nTGB+cSGlE5X8JXrJ7iwWyTj1jUcn/0NNrRTk9eOGD48Pkv+k4WQr+UsCqRCIJEB9YzMvvVVLeeV2fvdWkPxHDOrHx0qD5P9hJX9JIhUCkYg0NLbw0vrgm//v3tpJQ1MLIwb15aNnB8n/nMlK/tI7qBCI9KCGxhaWrN/Js5XV/G7dkeT/kbPHUlY8RslfeiUVApFuakv+5ZXBN//6xhbyB/blz88aS1lJIedOzlfyl15NhUDkAzjU1Jb8a3hx3Y725H/dmWNZEDb79MnOSnaYInFRIRCJU5D8a1lUWc2L63ZwsLGF4QP7cu0ZY1lQUsi5Sv6SolQIRLpwqKmF328Ikv9v1wbJf9iAHK45YwxlxWOYNUXJX1KfCoFIB4eaWli6oZbyympeXLeTA4eb25P//OJCZk3JJ0fJX9KICoEIQfL/w9u7KK/Yzm/D5D90QA4LSgqZX1zIeaco+Uv6irsQmNkAd6+PMhiRRDrc3MIfNuyiPGz22X+4mSH9cygrLmR+SSHnK/lLhjhhITCz84HHgEHABDP7EPC37v7ZqIMT6WmHm1v449u7KK+o5jcxyX9e8WjKSsYo+UtGiueK4N+Bq4CFAO7+ppldHGlUIj2oPflXhsn/UDODc/sw9/TRlJUUcv4pI+jbR8lfMldcTUPuvrXDfKct0YQj0jMam1v548ZayitqeGFtTXvyv2pmkPwvUPIXaRdPIdgaNg+5meUAnwfWRRuWyMlrbG7l5Y3BN/8X1tSw71Azebl9uLJoNAtKCrngVCV/kc7EUwg+DXyXYDL6bcALgPoHpFdobG7l5Xd2saiimsUxyf+KolHtyb9fn+xkhynSq8VTCE5z95tiV5jZBcDL0YQk0rWmlvCbf0U1L6zdwd6GJvL69eGKmaMoKy7kwqlK/iInI55C8D3grDjWiUSmqaWVP71TR3nFdl5Yu4M99WHyLxrF/OJCLpqm5C/yQR23EJjZecD5QIGZ3RWzaTDBHMQikWpqaWXZO3WUV1SzeG0Ne+qbGBSb/KeOIDdH/xVFuqurK4K+BM8O9AHyYtbvA66PMijJXM0trSzbFCT/59cEyX9g32yuKBpFWckYJX+RCBy3ELj774Hfm9kT7r4lgTFJhmluaWX5pvcpr9zO86tr2B0m/8uLgjb/i6cVKPmLRCiePoJ6M3sQmAnktq1098sii0rS3pHkH9zt8/7BRgb2zWbOjFGUlRRyiZK/SMLEUwieBJ4GFhDcSvpJoDbKoCQ9Nbe08srmIPk/vzpI/gPakn9xIZeepuQvkgzxFIJ8d/+hmX0+prloRdSBSXpoaXVe2VTXnvzrwuR/2fSRLCgp5NLTRir5iyRZPIWgKfyz2szKgO3A8OhCklTX0uq8srmORWHy33Wgkf452cyZMTL85j+S/n2V/EV6i3gKwTfMbAjwRYLnBwYDX4g0Kkk5La3Oq5vbOnx3sOvAYfrnZHNZmPxnK/mL9FonLATu/mz4ci8wG9qfLJYM19LqrHj3fcorqnludQ27DhwmNyeLOdOD+/xnTy9gQF/NfSTS23X1QFk28DGCMYaed/fVZrYA+AegP3BmYkKU3qSl1Vn5btDh+9zqGmr3B8n/sukjKSseo+QvkoK6+o39ITAeeBV4yMy2A6XAPe7+y0QEJ71Da6uzcstuyiu2syhM/v36hMm/JGj2GdhPyV8kVXX121sKlLh7q5nlAjXAKe5el5jQJJlaW51V7+2mvKKaRZXV7AyT/+zTguR/2XQlf5F00dVvcqO7twK4+yEz23SyRcDM5hIMYZ0NPObuD3Syz8eA+wAH3nT3T5zMMaTnxCb/51ZXs2NfkPwvPa2AspIxzFHyF0lLXf1WTzezivC1AaeEywa4u5d09cFhH8PDwBVAFbDCzBa6+9qYfaYCXwEucPfdZjayG+ciH0Brq/Pae7uDNv/KGmr2HaJvnywunVZAWUkhc2aMYpCSv0ha6+o3fEY3P/scYKO7bwIws6eAa4G1Mfv8DfCwu+8GcPed3TymxKG11Xl9627KK2pYVFndnvwvmVbAV0qmc9n0keTl5iQ7TBFJkK4GnevuQHNjga0xy1XAuR32mQZgZi8TNB/d5+7Pd/wgM7sNuA1gwoQJ3QwrMwXJfw+LKoM2/+q9h+ibncUlpxVwT/F05sxQ8hfJVMm+5u8DTAUuBcYBS82s2N33xO7k7o8CjwKUlpZ6ooNMVe5B8i+vqOa5ymq2h8n/4mkF/P3c05gzYxSDlfxFMl6UhWAbwe2nbcaF62JVAa+4exOw2cw2EBQGjWX0Abk7b7Ql/9U1bNvTECb/EXxJyV9EOhFXITCz/sAEd19/Ep+9AphqZpMJCsANQMc7gn4J3Aj8PzMbQdBUtOkkjiEEyf/Nqr3Bff6VQfLPyTYunlrAF6+cxuVFSv4icnwnLARmdjXwbwQzlk02szOA+939mq7e5+7NZnY7sJig/f9xd19jZvcDK919YbjtSjNbC7QAX9JzCvFxdyqq9lJeWU15RXV78r9oagF3XREk/yH9lfxF5MTMvesmdzNbBVwGLHH3M8N1le5enID4jlFaWuorV65MxqGTzt2p3LaX8opqyiurqdodJP8LTx1BWckYrlDyF5HjMLNV7l7a2ba4hqF2971mFrtOHbYJ0p78w7t9tr7fQJ8s48KpI/j8nKlcWTSaIQOU/EXkg4unEKwxs08A2eEDYHcCf4o2rMzm7qzeti9o9qnc3p78Lzh1BHdcNpUri0YxdEDfZIcpImkinkJwB3AvcBj4b4J2/W9EGVQmcnfWbN/X3ub/3vv1R5L/7KlcOVPJX0SiEU8hmO7u9xIUA+lBbcl/UWXQ5r+lrp7sMPl/bvYpXFk0mmEDlfxFJFrxFIJvm9lo4BngaXdfHXFMac3dWVu9r31Uz3fD5H/+Kfl89lIlfxFJvHhmKJsdFoKPAd83s8EEBUHNQ3Fyd9ZV76e8MrjPf/Oug+3J/9OXnMKVM0czXMlfRJIkrgfK3L2GYHKal4C/B76G+gm65O68VbO//Zv/pjD5nzcln9sunsJVSv4i0kvE80DZDODjwEeAOuBpgonspQN3Z/2OIPmXVwTJP8vgvFPy+dRFU7hq5ijyB/VLdpgiIkeJ54rgcYLkf5W7b484npTj7mzYcYDyiu08W1nNptog+c+aks+tF03mqpmjGaHkLyK9WDx9BOclIpBUs2HHfp6tqKa8YjvvhMn/3Mn5/PUFk5l7upK/iKSO4xYCM/uZu3/MzCo5+kniuGYoS0cb2pp9KqvZuPNAe/K/5YLJzJ05moI8JX8RST1dXRF8PvxzQSIC6a3e3rG//SGvt3cewAzOnTycT543k6tOH83IvNxkhygi0i1dzVBWHb78rLt/OXabmX0L+PKx70oPG3fup7yihvLK7WzYEST/cyYN55+uVfIXkfQTT2fxFRyb9Od1si6lbdx5IHjCt6Ka9Tv2YwYfnjSc+6+dydyZoxk5WMlfRNJTV30EnwE+C0wxs4qYTXnAy1EHFrVVW3bzXGU1Bw838/rWPbxVEyb/icP5x2tmMvf00YxS8heRDNDVFcF/A88B/wLcE7N+v7u/H2lUEVu1ZTc3/mA5jc2tAMwYncd9Vxcxr7hQyV9EMk5XhcDd/V0z+1zHDWY2PJWLwfJNdTSFRSDLYMGHxnDLBZOTHJWISHKc6IpgAbCK4PbR2JlpHJgSYVyRmjUlnz7ZRlOLk5Odxawp+ckOSUQkabKOt8HdF4R/Tnb3KeGfbT8pWwQAzp44jL+5KLgCeOjGMzl74rAkRyQikjzHLQRtzOwCMxsYvr7ZzL5jZhOiDy1aBeEtoOdMGp7kSEREkuuEhQD4T6DezD5EMNjcO8CPI40qAeobWwDo3zc7yZGIiCRXPIWg2d0duBb4D3d/mOAW0pTW0NhClkG/PvH8FYiIpK94Hijbb2ZfAf4CuMjMsoCcaMOKXn1jCwP69sHMTryziEgai+fr8McJJq7/63CCmnHAg5FGlQANTc1qFhIRIY5CECb/J4EhZrYAOOTu/xV5ZBELrghUCERE4rlr6GPAq8BHCeYtfsXMro86sKjVN7bQP0eFQEQknj6Ce4EPu/tOADMrAH4LPBNlYFFr0BWBiAgQXx9BVlsRCNXF+b5erb6xmQF946mDIiLpLZ5M+LyZLQZ+Gi5/HFgUXUiJUd/YoonkRUSIb87iL5nZnwMXhqsedfdfRBtW9Bqa1DQkIgJdz0cwFfg34BSgErjb3bclKrCo6a4hEZFAV239jwPPAh8hGIH0eyf74WY218zWm9lGM7uni/0+YmZuZqUne4wPqv5wM/1z1EcgItJVJsxz9x+Er9eb2Wsn88Fmlg08TDDVZRWwwswWuvvaDvvlAZ8HXjmZz+8Od6deTUMiIkDXhSDXzM7kyDwE/WOX3f1EheEcYKO7bwIws6cIxita22G/fwK+BXzpJGP/wA43t+KuAedERKDrQlANfCdmuSZm2YHLTvDZY4GtMctVwLmxO5jZWcB4dy83s+MWAjO7DbgNYMKE7o+A3TbyqK4IRES6KATuPjvKA4eD130HuOVE+7r7o8CjAKWlpd7dY9c3NgMqBCIiEO2DYduA8THL48J1bfKA04ElZvYuMAtYmIgO44b2uQjUWSwiEmUhWAFMNbPJZtYXuAFY2LbR3fe6+wh3n+Tuk4DlwDXuvjLCmICYpiGNNSQiEl0hcPdm4HZgMbAO+Jm7rzGz+83smqiOGw/1EYiIHHHCthELZm65CZji7veH8xWPdvdXT/Red19Eh+Eo3P1rx9n30rgi7gENTUEfge4aEhGJ74rg/wLnATeGy/sJng9IWW1XBAP7qY9ARCSeTHiuu59lZq8DuPvusM0/ZbVPXK8+AhGRuK4ImsKnhB3a5yNojTSqiDWoj0BEpF08heAh4BfASDP7JvBH4J8jjSpiRzqL1TQkIhLPMNRPmtkqYA7B8BLXufu6yCOLUENjM2aQm5Py8+uIiHRbPHcNTQDqgV/HrnP396IMLEpt8xUHN0SJiGS2eNpGygn6BwzIBSYD64GZEcYVKY08KiJyRDxNQ8Wxy+FAcZ+NLKIEaGhs0TMEIiKhk24kD4efPveEO/Zi9Y3NDNCkNCIiQHx9BHfFLGYBZwHbI4soAep1RSAi0i6er8V5Ma+bCfoM/ieacBKjQfMVi4i067IQhA+S5bn73QmKJyHqG1sYOiAn2WGIiPQKx+0jMLM+7t4CXJDAeBKivrFZcxGIiIS6yoavEvQHvGFmC4GfAwfbNrr7/0YcW2TqG1s0F4GISCier8W5QB3BHMVtzxM4kLKFQLePiogc0VUhGBneMbSaIwWgTbfnDU4Wd9cDZSIiMboqBNnAII4uAG1SthA0trTS0uoqBCIioa4KQbW735+wSBJEE9eLiBytqyeL03JENs1XLCJytK4KwZyERZFAKgQiIkc7biFw9/cTGUiiNGiaShGRo2TczCz1jc2AZicTEWmTeYWgKWwa6qcrAhERyMBCoInrRUSOlnGFoL2zWPMRiIgAGVgIGsI+Ag0xISISyLhCoNtHRUSOlrGFQLePiogEMq4QNDS1kJuTRVZWWj44LSJy0jKuENQ3NusZAhGRGJEWAjOba2brzWyjmd3Tyfa7zGytmVWY2YtmNjHKeCCcuF7NQiIi7SIrBOF8xw8D84Ai4EYzK+qw2+tAqbuXAM8A/xpVPG00cb2IyNGivCI4B9jo7pvcvRF4Crg2dgd3f8nd68PF5cC4COMB4KAKgYjIUaIsBGOBrTHLVeG647kVeK6zDWZ2m5mtNLOVtbW13QqqobFZzxCIiMToFZ3FZnYzUAo82Nl2d3/U3UvdvbSgoKBbx6pvbFFnsYhIjCgz4jZgfMzyuHDdUczscuBe4BJ3PxxhPIAmrhcR6SjKK4IVwFQzm2xmfYEbgIWxO5jZmcD3gWvcfWeEsbSrb2xhgO4aEhFpF1khcPdm4HZgMbAO+Jm7rzGz+83smnC3B4FBwM/N7A0zW3icj+sxwXMEKgQiIm0ibSx390XAog7rvhbz+vIoj9+ZhqYWTVwvIhKjV3QWJ0pTSytNLa4rAhGRGBlVCDTyqIjIsTKqELRPXK9CICLSLqMKwZGJ61UIRETaZFghaJuLQJ3FIiJtMqoQNDQFhWBgP10RiIi0yahCoM5iEZFjZVQhaJ+4Xk1DIiLtMqoQ6IpARORYKgQiIhkuowqBniMQETlWRhWCI1cE6iMQEWmTWYWgqZm+fbLIzrJkhyIi0mtkVCHQxPUiIsfKqEJw8LAmpRER6SijCkFDkyauFxHpKKMKgSauFxE5VsYVAl0RiIgcLaMKgTqLRUSOlVGFQBPXi4gcK6MKQUNjiwacExHpIKMKQX2TmoZERDrKrEKgPgIRkWNkTCFoaXUam1t115CISAcZUwg0cb2ISOcyphAcGYJancUiIrEyphC0D0GtsYZERI6ScYVgYD8VAhGRWBlTCBqawonr1TQkInKUjCkEmq9YRKRzGVcI+quPQETkKJEWAjOba2brzWyjmd3TyfZ+ZvZ0uP0VM5sUVSwNuiIQEelUZIXAzLKBh4F5QBFwo5kVddjtVmC3u58K/Dvwrajieat6HwDv7DwY1SFERFJSlFcE5wAb3X2TuzcCTwHXdtjnWuBH4etngDlm1uMzy6/aspvH/rgZgNt/+hqrtuzu6UOIiKSsKAvBWGBrzHJVuK7Tfdy9GdgL5Hf8IDO7zcxWmtnK2trakw5k+aY6WlodgOaWVpZvqjvpzxARSVcp0Vns7o+6e6m7lxYUFJz0+2dNyadfThbZBjl9spg15ZhaIyKSsaK8qX4bMD5meVy4rrN9qsysDzAE6PGv62dPHMaTn5rF8k11zJqSz9kTh/X0IUREUlaUhWAFMNXMJhMk/BuAT3TYZyHwSWAZcD3wO3f3KII5e+IwFQARkU5EVgjcvdnMbgcWA9nA4+6+xszuB1a6+0Lgh8CPzWwj8D5BsRARkQSKdLwFd18ELOqw7msxrw8BH40yBhER6VpKdBaLiEh0VAhERDKcCoGISIZTIRARyXAW0d2akTGzWmDLB3z7CGBXD4aTCnTOmUHnnBm6c84T3b3TJ3JTrhB0h56wsS4AAAe4SURBVJmtdPfSZMeRSDrnzKBzzgxRnbOahkREMpwKgYhIhsu0QvBosgNIAp1zZtA5Z4ZIzjmj+ghERORYmXZFICIiHagQiIhkuLQsBGY218zWm9lGM7unk+39zOzpcPsrZjYp8VH2rDjO+S4zW2tmFWb2oplNTEacPelE5xyz30fMzM0s5W81jOeczexj4b/1GjP770TH2NPi+L89wcxeMrPXw//f85MRZ08xs8fNbKeZrT7OdjOzh8K/jwozO6vbB3X3tPohGPL6HWAK0Bd4EyjqsM9ngUfC1zcATyc77gSc82xgQPj6M5lwzuF+ecBSYDlQmuy4E/DvPBV4HRgWLo9MdtwJOOdHgc+Er4uAd5MddzfP+WLgLGD1cbbPB54DDJgFvNLdY6bjFcE5wEZ33+TujcBTwLUd9rkW+FH4+hlgjplZAmPsaSc8Z3d/yd3rw8XlBDPGpbJ4/p0B/gn4FnAokcFFJJ5z/hvgYXffDeDuOxMcY0+L55wdGBy+HgJsT2B8Pc7dlxLMz3I81wL/5YHlwFAzK+zOMdOxEIwFtsYsV4XrOt3H3ZuBvUAqT2QczznHupXgG0UqO+E5h5fM4929PJGBRSief+dpwDQze9nMlpvZ3IRFF414zvk+4GYzqyKY/+SOxISWNCf7+35CkU5MI72Pmd0MlAKXJDuWKJlZFvAd4JYkh5JofQiahy4luOpbambF7r4nqVFF60bgCXf/tpmdRzDr4enu3prswFJFOl4RbAPGxyyPC9d1uo+Z9SG4nKxLSHTRiOecMbPLgXuBa9z9cIJii8qJzjkPOB1YYmbvErSlLkzxDuN4/p2rgIXu3uTum4ENBIUhVcVzzrcCPwNw92VALsHgbOkqrt/3k5GOhWAFMNXMJptZX4LO4IUd9lkIfDJ8fT3wOw97YVLUCc/ZzM4Evk9QBFK93RhOcM7uvtfdR7j7JHefRNAvco27r0xOuD0inv/bvyS4GsDMRhA0FW1KZJA9LJ5zfg+YA2BmMwgKQW1Co0yshcBfhncPzQL2unt1dz4w7ZqG3L3ZzG4HFhPccfC4u68xs/uBle6+EPghweXjRoJOmRuSF3H3xXnODwKDgJ+H/eLvufs1SQu6m+I857QS5zkvBq40s7VAC/Ald0/Zq904z/mLwA/M7O8IOo5vSeUvdmb2U4JiPiLs9/g6kAPg7o8Q9IPMBzYC9cBfdfuYKfz3JSIiPSAdm4ZEROQkqBCIiGQ4FQIRkQynQiAikuFUCEREMpwKgfRKZtZiZm/E/EzqYt8DPXC8J8xsc3is18InVE/2Mx4zs6Lw9T902Pan7sYYfk7b38tqM/u1mQ09wf5npPponBI93T4qvZKZHXD3QT29bxef8QTwrLs/Y2ZXAv/m7iXd+Lxux3SizzWzHwEb3P2bXex/C8Goq7f3dCySPnRFICnBzAaF8yi8ZmaVZnbMSKNmVmhmS2O+MV8Urr/SzJaF7/25mZ0oQS8FTg3fe1f4WavN7AvhuoFmVm5mb4brPx6uX2JmpWb2ANA/jOPJcNuB8M+nzKwsJuYnzOx6M8s2swfNbEU4xvzfxvHXsoxwsDEzOyc8x9fN7E9mdlr4JO79wMfDWD4exv64mb0a7tvZiK2SaZI99rZ+9NPZD8FTsW+EP78geAp+cLhtBMFTlW1XtAfCP78I3Bu+ziYYb2gEQWIfGK7/MvC1To73BHB9+PqjwCvA2UAlMJDgqew1wJnAR4AfxLx3SPjnEsI5D9piitmnLcY/A34Uvu5LMIpkf+A24Kvh+n7ASmByJ3EeiDm/nwNzw+XBQJ/w9eXA/4SvbwH+I+b9/wzcHL4eSjAW0cBk/3vrJ7k/aTfEhKSNBnc/o23BzHKAfzazi4FWgm/Co4CamPesAB4P9/2lu79hZpcQTFbycji0Rl+Cb9KdedDMvkowTs2tBOPX/MLdD4Yx/C9wEfA88G0z+xZBc9IfTuK8ngO+a2b9gLnAUndvCJujSszs+nC/IQSDxW3u8P7+ZvZGeP7rgN/E7P8jM5tKMMxCznGOfyVwjZndHS7nAhPCz5IMpUIgqeImoAA4292bLBhRNDd2B3dfGhaKMuAJM/sOsBv4jbvfGMcxvuTuz7QtmNmcznZy9w0WzHUwH/iGmb3o7vfHcxLufsjMlgBXAR8nmGgFgtmm7nD3xSf4iAZ3P8PMBhCMv/M54CGCCXhecvc/CzvWlxzn/QZ8xN3XxxOvZAb1EUiqGALsDIvAbOCYOZctmId5h7v/AHiMYLq/5cAFZtbW5j/QzKbFecw/ANeZ2QAzG0jQrPMHMxsD1Lv7TwgG8+tsztim8MqkM08TDBTWdnUBQVL/TNt7zGxaeMxOeTDb3J3AF+3IUOptQxHfErPrfoImsjaLgTssvDyyYFRayXAqBJIqngRKzawS+EvgrU72uRR408xeJ/i2/V13ryVIjD81swqCZqHp8RzQ3V8j6Dt4laDP4DF3fx0oBl4Nm2i+Dnyjk7c/ClS0dRZ38ALBxEC/9WD6RQgK11rgNQsmLf8+J7hiD2OpIJiY5V+BfwnPPfZ9LwFFbZ3FBFcOOWFsa8JlyXC6fVREJMPpikBEJMOpEIiIZDgVAhGRDKdCICKS4VQIREQynAqBiEiGUyEQEclw/x/ayELNWLdYQAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sdtl3O50_y2e",
        "colab_type": "text"
      },
      "source": [
        "По сравнению с бейзлайном на всех данных precision меток 0 и 1 снизился до 0.95 0.78, recall меток 0 и 1 снизился до 0.98 и 0.55, f1-score снизился до 0.94. Результаты по сравнению с top 1000 features улучшились. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bx7s7rgJ8mE7",
        "colab_type": "text"
      },
      "source": [
        "### Top 30000 features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0qMz0IL28qQT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "top30000_features = top_features[:30000]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u54tRADf8q0l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train_eli5 = X_train[:,top30000_features]\n",
        "X_test_eli5 = X_test[:,top30000_features]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mN-_iBV58q91",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "eli5_model = SGDClassifier().fit(X_train_eli5, y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CogYipJm8rIW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_pred = eli5_model.predict(X_test_eli5)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "niJiAosI8rVp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "887b3aef-9626-431d-9408-07c0c28d6fab"
      },
      "source": [
        "print(classification_report(y_test, y_pred))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.99      0.97     66765\n",
            "           1       0.83      0.54      0.65      7007\n",
            "\n",
            "    accuracy                           0.95     73772\n",
            "   macro avg       0.89      0.76      0.81     73772\n",
            "weighted avg       0.94      0.95      0.94     73772\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SDrl953pTQIv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "outputId": "43d9cf8d-43ab-419b-9a70-5bbb97d52b08"
      },
      "source": [
        "y_true = y_test\n",
        "y_predicted = y_pred \n",
        "fpr, tpr, _ = roc_curve(y_true, y_predicted)\n",
        "pyplot.plot(fpr, tpr, marker='.', label='Test')\n",
        "pyplot.xlabel('False Positive Rate')\n",
        "pyplot.ylabel('True Positive Rate')\n",
        "pyplot.legend()\n",
        "pyplot.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3hV9ZX/8fdKQhIu4ZbLEYFwkduJ1XqJAioKEqy1rTpTW6/tOL3Ym1pr7fzasU+nP6edsdNp56kdf2OtdexMnWrrTFs6tbWAImpFAaVeEkAEkaAmIdwvCSc56/fH3gmHEMJBsnNycj6v58nD2Zdz9toE9jrf/V37+zV3R0REcldepgMQEZHMUiIQEclxSgQiIjlOiUBEJMcpEYiI5LiCTAdwrMrKynzixImZDkNEJKusWrVqq7uXd7ct6xLBxIkTWblyZabDEBHJKma26UjbdGtIRCTHKRGIiOQ4JQIRkRyXdX0E3UkkEtTX19PS0pLpUHpdcXEx48aNY9CgQZkORUQGqAGRCOrr6ykpKWHixImYWabD6TXuTnNzM/X19UyaNCnT4YjIABXZrSEzu9/MGs3slSNsNzO7y8zWm9lLZnbGuz1WS0sLpaWlAyoJAJgZpaWlA7KlIyL9R5R9BA8AF/ew/f3A1PDnBuDfjudgAy0JdBio5yUix2bVpu3c/cR6Vm3a3uufHdmtIXdfZmYTe9jlMuA/PBgHe7mZjTSzMe7+dlQxiYhkG3fnkVX1fO1/XibpTmFBHg9+ahZnThjVa8fIZB/BWGBzynJ9uO6wRGBmNxC0GqisrOyT4I5Fc3Mz8+fPB+Cdd94hPz+f8vLgAb7nn3+ewsLCHt+/dOlSCgsLOeeccyKPVUT6v9a2dpZv2Mbi2gaW1DXw1s6Dt4cTbUmWb2geMIkgbe5+L3AvQHV1db+bSae0tJTVq1cD8M1vfpNhw4Zx2223pf3+pUuXMmzYMCUCkRy2fe8BnljbyOK6Bp5c28TeA+0MHpTPnKll/MUZY7nvqY20tScZVJDHrMmlvXrsTCaCLcD4lOVx4bo+sWrTdpZvaGbW5NJezaydn79qFbfeeit79uyhrKyMBx54gDFjxnDXXXdxzz33UFBQQFVVFXfeeSf33HMP+fn5/OxnP+OHP/whc+bM6fV4RKT/2bh1L4trG1hU18DKN7aRdKgoKeLS08ZyUVWM2SeVUjwoH4ALZ8Qiu2ZlMhEsBG40s4eAmcDO3ugf+L+/fZXat3b1uM/ulgRr3tlN0iHPYMYJJZQUH7lOv+rE4fzdh05OOwZ356abbuI3v/kN5eXlPPzww9x+++3cf//93HnnnWzcuJGioiJ27NjByJEj+exnP3vMrQgRyT7tSefFN7ezqK6BxbUNvN60F4D4mOHcOG8K8+MxThk7gry8w4tEzpwwKpIvrRBhIjCznwNzgTIzqwf+DhgE4O73AI8ClwDrgX3AX0cVS1e7WtpIhjeYkh4s95QIjlVrayuvvPIKCxYsAKC9vZ0xY8YAcOqpp3Lttddy+eWXc/nll/faMUWkf9rb2sZTr21lcV0Dj69pZNveAxTkGbNPKuXjsycyP17BuFFDMhpjlFVDVx9luwNf6O3jpvPNfdWm7Vx733ISbcH9th9cdXqvZlp35+STT+bZZ589bNvvfvc7li1bxm9/+1u+/e1v8/LLL/facUWkf2jY1cLi8Fv/M683c6AtyfDiAi6cUUFNVYzzp5UzvBe/fB6vrOgs7m1nThjFg5+aFdn9tqKiIpqamnj22WeZPXs2iUSCdevWEY/H2bx5M/PmzeO8887joYceYs+ePZSUlLBrV8+3s0Sk/3J36t7eHVz86xp4qX4nAJWjh/CxWROoiceonjiKQfn9c3i3nEwEEO39try8PB555BFuvvlmdu7cSVtbG7fccgvTpk3juuuuY+fOnbg7N998MyNHjuRDH/oQV1xxBb/5zW/UWSySJQ60JXluYzOLaxtYXNfIlh37MYPTx4/kby6ezoJ4jCkVw7LioVAL7tBkj+rqau86MU1dXR3xeDxDEUVvoJ+fSLbYse8AS9c2sSgs8dzT2kbxoDzmTC1nQTzGvBkVlJcUZTrMbpnZKnev7m5bzrYIRETSsal5L4tqg1s+K97YTnvSKS8p4kPvHUNNPMa5U8o6SzyzlRKBiEiK9qSzevOOzs7e1xr3AEGZ+ecuOImaqhinHqHEM1sNmETg7llxL+5YZdutO5FstO9AG0+nlHhu3ROUeM6cPJprZlZSE48xfnRmSzyjNCASQXFxMc3NzQNuKOqO+QiKi4szHYrIgNO4q4UlaxpZXNvA0+u30tqWpKS4gHnTgxLPC6aVM2Jw/ynxjNKASATjxo2jvr6epqamTIfS6zpmKBOR4+PurHlnd1Dls6aRP2/eAcD40YO5ZmYlC+Ixzpo0ut+WeEZpQCSCQYMGaQYvETnMgbYkz2/cxuK6BhbVNrBlx34AThs/kq+8bzo18RjTYtlR4hmlAZEIREQ67NyXYOm6RhbVBiWeu1vbKCrIY87UMm66cAoXxiuoKNHt1lRKBCKS9d5s3tc5kNvzb2yjPemUDSvkklPGUFMV47wpZQwuzO4SzygpEYhI1kkmndX1O8KnehtY1xCUeE6PlfDZCyYzPx7jtHEjB1SJZ5SUCEQkK+w/0M7T67cGs3ataWTrnlby84yZk0Zz1VlBiWdl6cAt8YySEoGI9FuNu1t4vC6Yteup18ISz6IC5s6ooCZewdxpFYwYkhslnlFSIhCRfsPdWdewp7PKZ3VY4jl25GCuPruSBVUxzpo4msKC3CvxjJISgYhkVKI9yYqN24LO3roGNm8LSjzfO34kt100jZqqGNNjJTlf4hklJQIR6XM79yd4cl0Ti2sbeGJtI7tbghLP86aU8fm5U5g/o4KK4Srx7CtKBCLSJzZv29c5cctzG7bRlnRKhxby/vecQE08xnlTyxhSqEtSJuhvXUQikUw6L23Z2Vniuead3QBMrRjGp8+fTE08xmnjR5KvEs+MUyIQkV7TkmjnmfVbw2/+jTTtDko8z5o4iq9/IE5NPMbEsqGZDlO6UCIQkePStLuVx9cEF/6nXmuiJZFkWFEBF0wPZu2aO72ckUMKMx2m9ECJQESOibvzWuOezlm7Vm/egXtQ4nll9XhqqmLMnFSqEs8sokQgIkeVaE+y4o1tLK4NHu56c9s+AE4dN4Iv1UyjJh4jPkYlntlKiUBEurWrJcGTa5tYXNfAE2sa2dXSRmFBHueeVMpnLpjM/BkxThihEs+BQIlARDpt3raPJWFH7/INzZ0lnu87+QTmx2PMmVrG0CJdNgYa/UZFclgy6by8ZWfnkA4dJZ5TKobxqTmTWVBVwWnjR6nEc4BTIhDJMS2Jdv70+lYW1TaypK6Bxt2t5BmcNXE0X/9AnPnxGJNU4plTlAhEcsDWPa08Hk7U/tRrW9mfaGdoYT5zp1dQUxWM4jlqqEo8c5USgcgA5O683rSHRWGVzwtvbscdThxRzEeqx1ETjzFz8miKCjRrlygRiAwYbe1JVm7a3jmkwxvNQYnnKWNHcMv8adRUVVA1ZrhKPOUwSgQiWWx3S4Jl64IhHR5f08jO/QkK8/M4Z0opn5ozmfnxCsaMGJzpMKWfizQRmNnFwA+AfOA+d7+zy/ZK4KfAyHCfr7r7o1HGJJLttuzYz5Kwymf5hmYS7c6oIYOoicdYUFXBeVPLGaYSTzkGkf1rMbN84G5gAVAPrDCzhe5em7Lb14FfuPu/mVkV8CgwMaqYRLKRu/PKll3BxC21DdS+vQuAyeVD+cS5k6ipinFGpUo85d2L8mvD2cB6d98AYGYPAZcBqYnAgeHh6xHAWxHGI5I1WhLtPLuhufN+f8OuoMSzesJo/vaSGcyPxzipfFimw5QBIspEMBbYnLJcD8zsss83gT+a2U3AUKCmuw8ysxuAGwAqKyt7PVCR/qA5LPFcUtfIstea2HegnSGF+VwwrZyaeIx5MyoYrRJPiUCmbyReDTzg7t8zs9nAf5rZe9w9mbqTu98L3AtQXV3tGYhTJBLrG4OJ2hfXNrAqLPE8YXgxf3nGWGriMWZNLqV4kEo8JVpRJoItwPiU5XHhulSfBC4GcPdnzawYKAMaI4xLJGPa2pOs2rS9c+KWjVv3AnDyicO5+cKpLKiKcfKJKvGUvhVlIlgBTDWzSQQJ4Crgmi77vAnMBx4wszhQDDRFGJNIn9vT2saycKL2x9c2smNfUOI5+6RSPnHeJObPqODEkSrxlMyJLBG4e5uZ3Qg8RlAaer+7v2pmdwAr3X0h8GXgx2b2JYKO4+vdXbd+JOu91VHiWdfI8tebOdCeZOSQQVw4o4IF8RhzpqnEU/oPy7brbnV1ta9cuTLTYYgcwt159a1dnbN2vfpWUOI5qWwoC6pi1MRjnFE5koJ8zdolmWFmq9y9urtt+koi8i61trXz7OvNLK5rYEldI2/vbCHP4MwJo/ja+2dQU6UST8kOSgQix2Db3gM8sSYYyG3Zuib2hiWe508t58sXxZg3vZzSYUWZDlPkmCgRiBzFhqaOEs9GVm7aRtIhNryIy08fS01VjNkq8ZQsp0Qg0kV70nnhzWAUz0V1DWxoCko8q8YM58YLp7IgHuM9Y1XiKQOHEoEIsLe1jadea2JRbSOPr2lg+74Eg/KNWZNLuf6cicyPxxirEk8ZoJQIJGe9vXM/S+qC+/1/Wh+UeI4YHJR41sRjnD+tjJLiQZkOUyRySgSSM9yd2rd3sTictevlLTsBmFA6hI/PnkBNVYzqCaNU4ik5R4lABrTWtnaWb9jGknA8n7d2tmAGZ1SO4v9cPIMFVRWcVD5M9/slpykRyICzfe8BnlgbfOt/cm1Q4jl4UD5zppZxy4JpXDijgjKVeIp0UiKQAWHj1r2dVT4r3whKPCtKirj0tLEsqKrgnJPKVOIpcgRpJwIzG+Lu+6IMRiRd7UnnxTe3d87a9XpY4jnjhBK+MG8KNfEYp4wdQZ5m7RI5qqMmAjM7B7gPGAZUmtl7gc+4++ejDk4kVVDieXCi9m17D1CQZ8w+qZSPzZrA/HiM8aOHZDpMkayTTovgX4D3AQsB3P3PZnZ+pFGJhBp2tXRO3PLM680caEsyvLggKPGsinH+tHKGq8RT5LikdWvI3Td3qapojyYcyXXuTt3bu8OJWxp4qT4o8awcPYSPzZpATTxG9cRRDFKJp0ivSScRbA5vD7mZDQK+CNRFG5bkkgNtSZ7b2DFReyNbduzHDE4fP5K/uXg6C+IxplSoxFMkKukkgs8CPyCYjH4L8EdA/QNyXHbsO8DStU0sCks897S2UTwojzlTy/ni/KnMm1FBeYlKPEX6QjqJYLq7X5u6wszOBZ6JJiQZqDY17+2cuGXFG9tpTzrlJUV86L1jqInHOHeKSjxFMiGdRPBD4Iw01okcoj3prN68o7Oz97XGPUBQ4vm5C06ipirGqSrxFMm4IyYCM5sNnAOUm9mtKZuGE8xBLHKYfQfaeDqlxHPrnqDEc+bk0Vwzs5IalXiK9Ds9tQgKCZ4dKABKUtbvAq6IMijJLo27WliyppHFtQ08vX4rrW1JSooLmDc9KPG8YFo5IwarxFOkvzpiInD3J4EnzewBd9/UhzFJP+furG3YHQ7p0MifN+8AYPzowVwzs5IF8RhnTRqtEk+RLJFOH8E+M/sucDJQ3LHS3S+MLCrpdw60JXl+47bO+v767fsBOG38SL7yvunUxGNMi6nEUyQbpZMIHgQeBj5IUEr6V0BTlEFJ/7BzX4Kl6xpZVBuUeO5ubaOoII85U8u4cd4ULoxXUFFSfPQPEpF+LZ1EUOruPzGzL6bcLloRdWCSGW827+scyG3FG9toSzplwwq55JQx1FTFOG9KGYMLVSsgMpCkkwgS4Z9vm9kHgLeA0dGFJH0pmXRW1+8In+ptYF1DUOI5PVbCZy6YzPx4jNPGjVSJp8gAlk4i+JaZjQC+TPD8wHDglkijkkjtP9DO0+u3sri2gSVrGtm6p5X8PGPmpNFcdVZQ4llZqhJPkVxx1ETg7v8bvtwJzIPOJ4slizTubuHxcKL2p14LSzyLCpg7o4KaeAVzp1UwYohKPEVyUU8PlOUDHyUYY+gP7v6KmX0Q+FtgMHB634Qo74a7s65hD4vrGlhU28DqsMRz7MjBXH12JQuqYpw1cTSFBSrxFMl1PbUIfgKMB54H7jKzt4Bq4Kvu/uu+CE6OTaI9yYqN24LO3roGNm8LSjzfO34kt100jZqqGNNjJSrxFJFD9JQIqoFT3T1pZsXAO8BJ7t7cN6FJOnbuT/DkuiYW1zbwxNpGdrcEJZ7nTSnj83OnMH9GBRXDVeIpIkfWUyI44O5JAHdvMbMNx5oEzOxigiGs84H73P3Obvb5KPBNwIE/u/s1x3KMXLR5277OB7ue2xCUeJYOLeT97zmBmniM86aWMaQw7emoRSTH9XS1mGFmL4WvDTgpXDbA3f3Unj447GO4G1gA1AMrzGyhu9em7DMV+BpwrrtvN7OK4ziXASuZdF7asrOzxHPNO7sBmFoxjE+fP5maeIzTxo8kXyWeIvIu9JQI4sf52WcD6919A4CZPQRcBtSm7PNp4G533w7g7o3HecwBoyXRzjPrt4bf/Btp2h2UeJ41cRRf/0CcmniMiWVDMx2miAwAPQ06d7wDzY0FNqcs1wMzu+wzDcDMniG4ffRNd/9D1w8ysxuAGwAqKyuPM6z+q2l3K0+saWRRXQNPvdZESyLJsKICLphezoJ4jLnTyxk5pDDTYYrIAJPpG8kFwFRgLjAOWGZmp7j7jtSd3P1e4F6A6upq7+sgo+LuvNa4p3Pilhc378A9KPG8sno8NVUxZk4qVYmniEQqykSwhaD8tMO4cF2qeuA5d08AG81sHUFiGLBjGSXak6x4YxuLa4OHu97ctg+AU8eN4Es106iJx4iPUYmniPSdtBKBmQ0GKt197TF89gpgqplNIkgAVwFdK4J+DVwN/LuZlRHcKtpwDMfICrtaEjy5tonFdQ08saaRXS1tFBbkce5JpcF4PjNinDBCJZ4ikhlHTQRm9iHgnwlmLJtkZqcBd7j7pT29z93bzOxG4DGC+//3u/urZnYHsNLdF4bbLjKzWqAd+MpAeU5h87Z9LAk7epdvaKYt6YweWshFJwclnnOmljG0KNN35kREwNx7vuVuZquAC4Gl7n56uO5ldz+lD+I7THV1ta9cuTITh+5RMum8vGVn55AOHSWeUyqGMT9ewYJ4jNMrR6nEU0QywsxWuXt1d9vSGoba3Xd2uWc9YDpsj0dLop0/vb6VRbWNLKlroHF3K3kGZ00czdc/EGd+PMYklXiKSD+XTiJ41cyuAfLDB8BuBv4UbVj919Y9rTweTtT+1Gtb2Z9oZ2hhPnOnV1BTFYziOWqoSjxFJHukkwhuAm4HWoH/Iriv/60og+pP3J3Xm/awKKzyeeHN7bjDiSOK+Uj1OGriMWZOHk1RgWbtEpHslE4imOHutxMkg5zQ1p5k5abtnUM6vNEclHieMnYEt8yfRk1VBVVjhqvEU0QGhHQSwffM7ATgEeBhd38l4pgyYndLgmXrgiEdHl/TyM79CQrz8zhnSimfmjOZ+fEKxowYnOkwRUR6XTozlM0LE8FHgR+Z2XCChJDVt4dWbdrOH2vfoT3prH1nN8s3NJNod0YNGURNPMaCqgrOm1rOMJV4isgAl9ZVzt3fIZic5gngb4BvkMX9BKs2becj9/yJZFj7dOLIYj5x7iRqqmKcoRJPEckx6TxQFgeuBD4MNAMPE0xkn7WWb2juTAJ5BtfOnMAX5k3JbFAiIhmSTovgfoKL//vc/a2I4+kTsyaXAsHECoUFeZ3LIiK5KJ0+gtl9EUhfOqNyJACzTyrlyxdN58wJozIckYhI5hwxEZjZL9z9o2b2Moc+SZzWDGX9WWtbEoDzppYpCYhIzuupRfDF8M8P9kUgfakl0Q5AsR4CExHhiDOeuPvb4cvPu/um1B/g830TXjQ6WgTFg5QIRETSmfpqQTfr3t/bgfSljhZBkWb+EhHpsY/gcwTf/Ceb2Uspm0qAZ6IOLEpqEYiIHNRTH8F/Ab8H/hH4asr63e6+LdKoIqYWgYjIQT0lAnf3N8zsC103mNnobE4GahGIiBx0tBbBB4FVBOWjqeMuODA5wrgi1dkiGKQWgYjIEROBu38w/HNS34XTN1oSYYtA5aMiIkevGjKzc81saPj6OjP7vplVRh9adFrb1CIQEemQzpXw34B9ZvZegsHmXgf+M9KoIqYWgYjIQekkgjZ3d+Ay4F/d/W6CEtKs1dEiKFaLQEQkrdFHd5vZ14CPAXPMLA8YFG1Y0epoEWieYRGR9FoEVxJMXP+JcIKaccB3I40qYuojEBE56KhXwvDi/yAwwsw+CLS4+39EHlmEDrYIlAhERNKpGvoo8DzwEYJ5i58zsyuiDixKrW3tFBXkYaYpKUVE0ukjuB04y90bAcysHFgMPBJlYFFqTSTVGhARCaVzNczrSAKh5jTf12+1JNo1vISISCidFsEfzOwx4Ofh8pXAo9GFFL3WtqQ6ikVEQunMWfwVM/tL4Lxw1b3u/qtow4pWS6JdD5OJiIR6mo9gKvDPwEnAy8Bt7r6lrwKLUmtbUreGRERCPd0fuR/4X+DDBCOQ/vBYP9zMLjaztWa23sy+2sN+HzYzN7PqYz3Gu9GSaFdnsYhIqKdbQyXu/uPw9Voze+FYPtjM8oG7Caa6rAdWmNlCd6/tsl8J8EXguWP5/OPR2pZksFoEIiJAzy2CYjM73czOMLMzgMFdlo/mbGC9u29w9wPAQwTjFXX198B3gJZjjv5dUotAROSgnloEbwPfT1l+J2XZgQuP8tljgc0py/XAzNQdwoQy3t1/Z2ZfOdIHmdkNwA0AlZXHPwK2+ghERA7qaWKaeVEeOBy87vvA9Ufb193vBe4FqK6u9uM9tloEIiIHRXk13AKMT1keF67rUAK8B1hqZm8As4CFfdFh3JJIUqQWgYgIEG0iWAFMNbNJZlYIXAUs7Njo7jvdvczdJ7r7RGA5cKm7r4wwJuDgWEMiIhJhInD3NuBG4DGgDviFu79qZneY2aVRHTcdrQn1EYiIdDjqk8UWDNF5LTDZ3e8I5ys+wd2fP9p73f1RugxH4e7fOMK+c9OK+Dglk86B9qRmJxMRCaVzNfx/wGzg6nB5N8HzAVmptU2zk4mIpEpn0LmZ7n6Gmb0I4O7bw3v+WUnzFYuIHCqdq2EifErYoXM+gmSkUUVI8xWLiBwqnURwF/AroMLMvg08DfxDpFFFSC0CEZFDpTMM9YNmtgqYDxhwubvXRR5ZRNQiEBE5VDpVQ5XAPuC3qevc/c0oA4tKS0ItAhGRVOl0Fv+OoH/AgGJgErAWODnCuCKjqiERkUOlc2volNTlcKC4z0cWUcTUIhAROdQxXw3d/QW6jCKaTTpaBHqyWEQkkE4fwa0pi3nAGcBbkUUUsY4WgcYaEhEJpNNHUJLyuo2gz+C/owknemoRiIgcqsdEED5IVuLut/VRPJFTi0BE5FBHvBqaWYG7twPn9mE8ketMBGoRiIgAPbcInifoD1htZguBXwJ7Oza6+/9EHFskDpaPqkUgIgLp9REUA80EcxR3PE/gQHYmAt0aEhE5RE+JoCKsGHqFgwmgw3HPG5wprW1JigryCKZZEBGRnhJBPjCMQxNAh6xNBC2JdlUMiYik6CkRvO3ud/RZJH2ktU2zk4mIpOrpijgg7520JNo1zpCISIqeEsH8PouiD6lFICJyqCNeEd19W18G0lfUIhAROVTOfTVuSahFICKSKueuiK1tahGIiKTKuUSgFoGIyKFy7oqoFoGIyKFyLhG0JJIUqUUgItIp566IQfmoWgQiIh1yLxEk2jXgnIhIipy7IqpFICJyqJxKBO1J50B7Ui0CEZEUkV4RzexiM1trZuvN7KvdbL/VzGrN7CUzW2JmE6KMp7UtmItALQIRkYMiSwThfMd3A+8HqoCrzayqy24vAtXufirwCPBPUcUD0JrQ7GQiIl1FeUU8G1jv7hvc/QDwEHBZ6g7u/oS77wsXlwPjIoyHFrUIREQOE2UiGAtsTlmuD9cdySeB33e3wcxuMLOVZrayqanpXQekFoGIyOH6xRXRzK4DqoHvdrfd3e9192p3ry4vL3/Xx1GLQETkcOlMXv9ubQHGpyyPC9cdwsxqgNuBC9y9NcJ4OlsEGmtIROSgKK+IK4CpZjbJzAqBq4CFqTuY2enAj4BL3b0xwliAYC4CQGMNiYikiCwRuHsbcCPwGFAH/MLdXzWzO8zs0nC37wLDgF+a2WozW3iEj+sVrW1qEYiIdBXlrSHc/VHg0S7rvpHyuibK43elFoGIyOFy6qtxi1oEIiKHyakrYqtaBCIih8mpRNDRItB8BCIiB+XUFVEtAhGRw+VWIlAfgYjIYXLqitiaaMcMCvNz6rRFRHqUU1fElrZgLgIzy3QoIiL9Rk4lgtZEu8YZEhHpIqcSQUtCs5OJiHSVU1fFlja1CEREusqpRNCqFoGIyGFy6qqoFoGIyOFyKhGoRSAicricuiqqRSAicricSgRqEYiIHC6nrootbe0UqUUgInKInEoErYkkxRpwTkTkELmVCNraNQS1iEgXOXVVbFGLQETkMDmVCNQiEBE5XM5cFduTTqLd1SIQEekiZxJBa1s4O5laBCIih8iZq2JLIpydTM8RiIgcImeuigdbBLo1JCKSKmcSQWeLQLeGREQOkTNXxZZE0CJQZ7GIyKFyJhG0tgUtAnUWi4gcKmeuimoRiIh0L2cSgVoEIiLdy5mrYkeLoEgtAhGRQ+RMIuhoEahqSETkUJFeFc3sYjNba2brzeyr3WwvMrOHw+3PmdnEqGJRi0BEpHuRJQIzywfuBt4PVAFXm1lVl90+CWx39ynAvwDfiSqe1xr3ALC2YXdUhxARyUpRtgjOBta7+wZ3PwA8BFzWZZ/LgJ+Grx8B5puZ9XYgqzZt59+f3gjAjQ++wKpN23v7ECIiWSvKRDAW2JyyXB+u63Yfd28DdgKlXT/IzG4ws5VmtrKpqemYA1m+oZn2pAOQaE+yfEPzMX+GiMhAlTnyFHgAAAi+SURBVBU9p+5+r7tXu3t1eXn5Mb9/1uRSigblkW8wqCCPWZMPyzUiIjmrIMLP3gKMT1keF67rbp96MysARgC9/nX9zAmjePBTs1i+oZlZk0s5c8Ko3j6EiEjWijIRrACmmtkkggv+VcA1XfZZCPwV8CxwBfC4u3sUwZw5YZQSgIhINyJLBO7eZmY3Ao8B+cD97v6qmd0BrHT3hcBPgP80s/XANoJkISIifSjKFgHu/ijwaJd130h53QJ8JMoYRESkZ1nRWSwiItFRIhARyXFKBCIiOU6JQEQkx1lE1ZqRMbMmYNO7fHsZsLUXw8kGOufcoHPODcdzzhPcvdsncrMuERwPM1vp7tWZjqMv6Zxzg845N0R1zro1JCKS45QIRERyXK4lgnszHUAG6Jxzg845N0RyzjnVRyAiIofLtRaBiIh0oUQgIpLjBmQiMLOLzWytma03s692s73IzB4Otz9nZhP7PsrelcY532pmtWb2kpktMbMJmYizNx3tnFP2+7CZuZllfalhOudsZh8Nf9evmtl/9XWMvS2Nf9uVZvaEmb0Y/vu+JBNx9hYzu9/MGs3slSNsNzO7K/z7eMnMzjjug7r7gPohGPL6dWAyUAj8Gajqss/ngXvC11cBD2c67j4453nAkPD153LhnMP9SoBlwHKgOtNx98HveSrwIjAqXK7IdNx9cM73Ap8LX1cBb2Q67uM85/OBM4BXjrD9EuD3gAGzgOeO95gDsUVwNrDe3Te4+wHgIeCyLvtcBvw0fP0IMN/MrA9j7G1HPWd3f8Ld94WLywlmjMtm6fyeAf4e+A7Q0pfBRSSdc/40cLe7bwdw98Y+jrG3pXPODgwPX48A3urD+Hqduy8jmJ/lSC4D/sMDy4GRZjbmeI45EBPBWGBzynJ9uK7bfdy9DdgJZPNExumcc6pPEnyjyGZHPeewyTze3X/Xl4FFKJ3f8zRgmpk9Y2bLzeziPosuGumc8zeB68ysnmD+k5v6JrSMOdb/70cV6cQ00v+Y2XVANXBBpmOJkpnlAd8Hrs9wKH2tgOD20FyCVt8yMzvF3XdkNKpoXQ084O7fM7PZBLMevsfdk5kOLFsMxBbBFmB8yvK4cF23+5hZAUFzsrlPootGOueMmdUAtwOXuntrH8UWlaOdcwnwHmCpmb1BcC91YZZ3GKfze64HFrp7wt03AusIEkO2SuecPwn8AsDdnwWKCQZnG6jS+v9+LAZiIlgBTDWzSWZWSNAZvLDLPguBvwpfXwE87mEvTJY66jmb2enAjwiSQLbfN4ajnLO773T3Mnef6O4TCfpFLnX3lZkJt1ek82/71wStAcysjOBW0Ya+DLKXpXPObwLzAcwsTpAImvo0yr61EPh4WD00C9jp7m8fzwcOuFtD7t5mZjcCjxFUHNzv7q+a2R3ASndfCPyEoPm4nqBT5qrMRXz80jzn7wLDgF+G/eJvuvulGQv6OKV5zgNKmuf8GHCRmdUC7cBX3D1rW7tpnvOXgR+b2ZcIOo6vz+Yvdmb2c4JkXhb2e/wdMAjA3e8h6Ae5BFgP7AP++riPmcV/XyIi0gsG4q0hERE5BkoEIiI5TolARCTHKRGIiOQ4JQIRkRynRCD9kpm1m9nqlJ+JPey7pxeO94CZbQyP9UL4hOqxfsZ9ZlYVvv7bLtv+dLwxhp/T8ffyipn91sxGHmX/07J9NE6JnspHpV8ysz3uPqy39+3hMx4A/tfdHzGzi4B/dvdTj+Pzjjumo32umf0UWOfu3+5h/+sJRl29sbdjkYFDLQLJCmY2LJxH4QUze9nMDhtp1MzGmNmylG/Mc8L1F5nZs+F7f2lmR7tALwOmhO+9NfysV8zslnDdUDP7nZn9OVx/Zbh+qZlVm9mdwOAwjgfDbXvCPx8ysw+kxPyAmV1hZvlm9l0zWxGOMf+ZNP5aniUcbMzMzg7P8UUz+5OZTQ+fxL0DuDKM5cow9vvN7Plw3+5GbJVck+mxt/Wjn+5+CJ6KXR3+/IrgKfjh4bYygqcqO1q0e8I/vwzcHr7OJxhvqIzgwj40XP9/gG90c7wHgCvC1x8BngPOBF4GhhI8lf0qcDrwYeDHKe8dEf65lHDOg46YUvbpiPEvgJ+GrwsJRpEcDNwAfD1cXwSsBCZ1E+eelPP7JXBxuDwcKAhf1wD/Hb6+HvjXlPf/A3Bd+HokwVhEQzP9+9ZPZn8G3BATMmDsd/fTOhbMbBDwD2Z2PpAk+CYcA95Jec8K4P5w31+7+2ozu4BgspJnwqE1Cgm+SXfnu2b2dYJxaj5JMH7Nr9x9bxjD/wBzgD8A3zOz7xDcTnrqGM7r98APzKwIuBhY5u77w9tRp5rZFeF+IwgGi9vY5f2DzWx1eP51wKKU/X9qZlMJhlkYdITjXwRcama3hcvFQGX4WZKjlAgkW1wLlANnunvCghFFi1N3cPdlYaL4APCAmX0f2A4scver0zjGV9z9kY4FM5vf3U7uvs6CuQ4uAb5lZkvc/Y50TsLdW8xsKfA+4EqCiVYgmG3qJnd/7Cgfsd/dTzOzIQTj73wBuItgAp4n3P0vwo71pUd4vwEfdve16cQruUF9BJItRgCNYRKYBxw257IF8zA3uPuPgfsIpvtbDpxrZh33/Iea2bQ0j/kUcLmZDTGzoQS3dZ4ysxOBfe7+M4LB/LqbMzYRtky68zDBQGEdrQsILuqf63iPmU0Lj9ktD2abuxn4sh0cSr1jKOLrU3bdTXCLrMNjwE0WNo8sGJVWcpwSgWSLB4FqM3sZ+Diwppt95gJ/NrMXCb5t/8DdmwgujD83s5cIbgvNSOeA7v4CQd/B8wR9Bve5+4vAKcDz4S2avwO+1c3b7wVe6ugs7uKPBBMDLfZg+kUIElct8IIFk5b/iKO02MNYXiKYmOWfgH8Mzz31fU8AVR2dxQQth0FhbK+Gy5LjVD4qIpLj1CIQEclxSgQiIjlOiUBEJMcpEYiI5DglAhGRHKdEICKS45QIRERy3P8H4OQkLIcRhoIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2iFfNL2epzT1",
        "colab_type": "text"
      },
      "source": [
        "По сравнению с бейзлайном на всех данных precision меток 0 и 1 снизился до 0.95 и 0.81, recall метки 1 снизился до 0.54, f1-score снизился до 0.94. Результаты по сравнению с top 10000 features улучшились."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RqS3w13T8_dS",
        "colab_type": "text"
      },
      "source": [
        "### Top 50000 features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TAaj6tMI9Gt_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "top50000_features = top_features[:50000]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4B6o6uue9G2G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train_eli5 = X_train[:,top50000_features]\n",
        "X_test_eli5 = X_test[:,top50000_features]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lLfHvc0s9HBU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "eli5_model = SGDClassifier().fit(X_train_eli5, y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jlBj7NG69G94",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_pred = eli5_model.predict(X_test_eli5)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KfjrONQ49G6e",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "e69aafac-e737-48b7-80aa-1c1a71a1a737"
      },
      "source": [
        "print(classification_report(y_test, y_pred))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.99      0.97     66765\n",
            "           1       0.81      0.55      0.65      7007\n",
            "\n",
            "    accuracy                           0.94     73772\n",
            "   macro avg       0.88      0.77      0.81     73772\n",
            "weighted avg       0.94      0.94      0.94     73772\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6dvjBco5Tu9C",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "outputId": "6821f737-4c7a-4a19-e78d-188511c3011f"
      },
      "source": [
        "y_true = y_test\n",
        "y_predicted = y_pred \n",
        "fpr, tpr, _ = roc_curve(y_true, y_predicted)\n",
        "pyplot.plot(fpr, tpr, marker='.', label='Test')\n",
        "pyplot.xlabel('False Positive Rate')\n",
        "pyplot.ylabel('True Positive Rate')\n",
        "pyplot.legend()\n",
        "pyplot.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxc9Xnv8c+j3Yu8SRpjvMubZLYAApsdS5AASYA2JECgCc1ClgJJILmXlLxSLk1a0jTpK6TcEpJwSRsSSGiTOA0Jbb1gNoNtdpBsvGAssC1Zlncto5nn/nGOxFjI8hjraDSa7/v10stzzpyZ8xzLnmd+y/k95u6IiEjuyst0ACIikllKBCIiOU6JQEQkxykRiIjkOCUCEZEcV5DpAI5UeXm5z5gxI9NhiIhklTVr1uxw94q+nsu6RDBjxgxWr16d6TBERLKKmW0+1HPqGhIRyXFKBCIiOU6JQEQkx2XdGEFf4vE4jY2NtLe3ZzqUAVdSUsKUKVMoLCzMdCgiMkwNi0TQ2NhIaWkpM2bMwMwyHc6AcXdaWlpobGxk5syZmQ5HRIapyLqGzOw+M2sys1cO8byZ2V1mtt7MXjKzU97rudrb2ykrKxtWSQDAzCgrKxuWLR0RGTqiHCO4H7ion+cvBuaEP9cD/3I0JxtuSaDbcL0uETkyaza3cvey9azZ3Drg7x1Z15C7rzCzGf0cchnwrx6sg73SzMaZ2SR33xpVTCIi2cbdeXhNI1//j5dJulNUkMcDn1nIqdPHD9g5MjlGMBnYkrLdGO57VyIws+sJWg1MmzZtUII7Ei0tLdTV1QGwbds28vPzqagIbuB79tlnKSoq6vf1y5cvp6ioiDPPPDPyWEVk6DvQ2cWT61tY2rCdpQ1NbN/T0fNcvCvJyo0twyYRpM3d7wXuBaipqRlylXTKysp44YUXALj99tsZPXo0X/3qV9N+/fLlyxk9erQSgUgOa2w9wLKGJpY0NPHUhhY6u5KUFhdw7twKKitGce+KjXQlkhQW5LGwsmxAz53JRPAWMDVle0q4b1Cs2dzKyo0tLKwsG9DM2vP+a9Zw8803s2/fPsrLy7n//vuZNGkSd911F/fccw8FBQXMnz+fO++8k3vuuYf8/Hx+/vOf88Mf/pBzzjlnwOMRkaElkXSef7OVJQ1NLK1vYu32vQDMKBvJXyycTl1VjJoZEygqCIZyz58Xi+wzK5OJYDFwg5k9CCwAdg/E+MD/+f2rvPb2nn6P2dsep2HbXpIOeQZVx5RSWnLoefrzjx3D33z4uLRjcHduvPFGfve731FRUcFDDz3Ebbfdxn333cedd97Jpk2bKC4uZteuXYwbN47Pf/7zR9yKEJHss7stzop1zSxtaGL52iZaD8QpyDNOmzGBb3ywmtqqGJUVo/t87anTx0fypRUiTARm9kvgfKDczBqBvwEKAdz9HuAR4BJgPXAA+MuoYultT3sXybCDKenBdn+J4Eh1dHTwyiuvcOGFFwKQSCSYNGkSACeeeCLXXHMNl19+OZdffvmAnVNEhh53Z+OO/Sytb2JJw3ZWvdFKIumMH1nIonkxaqtjnDOngrEjMnvDaJSzhq4+zPMO/NVAnzedb+5rNrdyzU9WEu8K+tt+cNXJA5pp3Z3jjjuOp59++l3P/eEPf2DFihX8/ve/59vf/jYvv/zygJ1XRDKvsyvJqjd2sqS+iaUN23mj5QAQ9Dx87txK6qpjvG/qePLzhs7U8KwYLB5op04fzwOfWRhZf1txcTHNzc08/fTTnHHGGcTjcdatW0d1dTVbtmxh0aJFnH322Tz44IPs27eP0tJS9uzpvztLRIauHfs6WL62maUN21mxbgf7OrooKsjjzFllfPrsmSyqijFl/MhMh3lIOZkIINr+try8PB5++GFuuukmdu/eTVdXF1/+8peZO3cu1157Lbt378bduemmmxg3bhwf/vCHueKKK/jd736nwWKRLODu1G/dy9KG7SxpaOKFLbtwh4ljivnwScdSWxXjrNlljCzKjo9YC3poskdNTY33LkxTX19PdXV1hiKK3nC/PpFs0B5P8NSGHWGXTxNbdwdLv5w0dRx1VTFqq2Icd+yYIbsagJmtcfeavp7LjnQlIpIBW3e3sTSc3vnkhh20x5OMKsrnnDkVfOXCGOfPqyBWWpLpMI+aEoGISCiRdF5s3BXO8mmifmswdjd1wgiuOm0addUxTp85geKC/AxHOrCGTSJw9yHbJDsa2dZ1J5Jt9rbHefz1oMtn+domWvZ3kp9nnDp9PF+/uIq66hizKkYPy8+XbsMiEZSUlNDS0jLslqLurkdQUpL9TU+RoeSNHfuDO3obtvPspp3EE87YEYWcP6+C2qoY582tYNzI/tcIG06GRSKYMmUKjY2NNDc3ZzqUAdddoUxE3rt4IsnqN1p7ZvlsbN4PwJzYaD519kzqqiZyyrRxFOTnZvXeYZEICgsLVcFLRA7Sur+T5euaWFLfxGPrmtnb3kVRfh4LKifwiYXTqaueyNQJQ3du/2AaFolARMTdWbd9H0satrO0vonn3mwl6VA+uphLjp/EoqoYZ88pZ3SxPvZ609+IiGSt9niClRtbWNoQfPN/a1cbACdMHsuNtXOoq45x/LFjyRtCyzkMRUoEIpJVtu9p71m3/4nXd9AWTzCiMJ+z55RzY+1sFlXFmDhGEyyOhBKBiAxpyaTzytu7+Z9wEbdX3grm9k8eN4KP1kyhtirGwsoySgqH19z+waREICJDzv6OLh5/fUdYqrGZHfs6yDM4Zdp4/tdF86irmsjcicN7bv9gUiIQkSFhy84DLKkPpnc+s3EnnYkkpSUFnDe3grrqGOfNjTFhVO7M7R9MSgQikhFdiSTPvbmrZ5bP6037AKisGMUnz5xObdVEamaMpzBH5/YPJiUCERk0uw/EWb6uKSzV2MzutqBU44LKCVx1+jRqq2LMLB+V6TBzjhKBiETG3dnQvI8l4SJuazYHpRrLRhVx4fyJ1IVz+weyVKwcOSUCERlQHV0Jnt20s2fd/jd3BqUa508awxfPn0VtVYyTpozT3P4hRIlARI5a894Olq0N1u1//PVm9ncmKC7I4+zZ5XzuvEoWzYtx7LgRmQ5TDkGJQESOmLvz6tt7gjt6G5p4ccsuACaNLeHykydTVx3jjMpyRhRpbn82UCIQkbQc6OziyfUt4dz+Jrbv6cAM3jd1HF99/1xqqyZSPalUc/uzkBKBiBxSY+uBnuUcntrQQmdXktHFBZw7t5zaqomcP6+C8tHFmQ5TjpISgYj0SCSdF7a09gz0NmzbC8D0spFcu2A6ddUxTpsxgaICze0fTpQIRHLcnvY4K9Y1s7S+iWVrm2g9ECc/zzhtxnhuu6Sa2uoYleWj1OUzjCkRiOSgjc37epZuXvXGTrqSzriRhSyaF6O2Ksa5cysYO0Jz+3OFEoFIDujsSrL6jZ1hnd4mNu0ISjVWHVPKZ8+tpK4qxsnTxpOvuf05SYlAZJhq2dfB8rXNLG1oYsW6ZvZ2dFFUkMeZs8r41FkzWFQVY8p4lWoUJQKRYcPdqd+6l2Vrm1hSv53nt+zCHWKlxXzopEnUVk3krNlljCzSf3s5mP5FiGSx9niCpzbs6Jnls3V3OwAnTRnLl+vmUlcd47hjx2igV/qlRCCSZbbubmNpQ7Ccw5MbdtAeTzKyKJ9z5pTzlQvmcn5VBbFSlWqU9EWaCMzsIuAHQD7wE3e/s9fz04CfAePCY25190eijEkk2ySTzouNu3pm+by2NSjVOHXCCK46LVi6eUHlBIoLtJyDvDeRJQIzywfuBi4EGoFVZrbY3V9LOewbwK/c/V/MbD7wCDAjqphEssXe9jhPvL6DJQ1NLF/bxI59neQZ1EyfwK0XV1FXFWN2TKUaZWBE2SI4HVjv7hsBzOxB4DIgNRE4MCZ8PBZ4O8J4RIa0zS37e/r6n9nUQjzhjCkp4Px5sbBUYwXjRqpUowy8KBPBZGBLynYjsKDXMbcD/2VmNwKjgAv6eiMzux64HmDatGkDHqhIJsQTSdZsbg27fLazoTmY2z8nNppPnT2T2nkxTp0+ngKVapSIZXqw+Grgfnf/npmdAfybmR3v7snUg9z9XuBegJqaGs9AnCIDonV/J4+ta2ZJQxOPrW1iT3sXRfl5LKicwF8sDOr0TivT3H4ZXFEmgreAqSnbU8J9qT4NXATg7k+bWQlQDjRFGJfIoHF31m0PlnNY2rCdNZtbSTqUjy7mouOPobZqImfPKWd0caa/k0kui/Jf3ypgjpnNJEgAVwEf73XMm0AdcL+ZVQMlQHOEMYlErj2eYOXGlp5ZPm/tagPg+MljuKF2DnVVMU6YPFalGmXIiCwRuHuXmd0APEowNfQ+d3/VzO4AVrv7YuAW4Mdm9hWCgePr3F1dP5J1mva091TreuL1HbTFE5QU5nH27ApuqJ3Nonkxjhmruf0yNEXaHg3vCXik175vpjx+DTgryhhEopBMOq+8vbtnls/Lb+0GYPK4EVxx6hRqq2OcUVlGSaHm9svQp45JkTTt7+jiifU7WFrfxNK1TTTvDUo1njJtPF/7wDzqqmPMm6hSjZJ9lAhE+rFl54GeLp+VG1roTCQpLS7g3HkV1FXFOH9ejAmjNLdfspsSgUiKrkSS57fsCrt8trNu+z4AKitG8ckzg+mdNTPGU6i5/TKMKBFIztt9IM5jrzeztH47y9c1s+tAnII8Y0HlBK4M1/KZWT4q02GKREaJQHKOu7OheV/PQO/qza0kks6EUUXUVU2krjrG2XPKGVOiUo2SG5QIJCd0diV5ZlNLz4f/mzsPAFA9aQxfOG8WtdUxTpoyTqUaJScpEciw1by3g2Vrg3X7H3+9mf2dCYoL8jhrdjnXn1tJbVWMY8eNyHSYIhmnRCDDhrvz6tt7emb5vLhlFwDHjCnhspMnU1cV48xZ5Ywo0tx+kVRKBJLV2joTPLk+WLd/WUMT2/a0YwYnTRnHLRfOpbY6xvxJKtUo0h8lAsk6b+3qLtW4nac2tNDRlWRUUT7nzq2gNpzbX1FanOkwRbJG2onAzEa6+4EogxHpSyLpvLBlF0sbtrOkvomGbXsBmDZhJB9fMI26qomcPnMCRQWa2y/yXhw2EZjZmcBPgNHANDM7Cficu38x6uAkd+1pj/P4uh0sadjO8rXN7NzfSX6ecdqM8fz1JVXUVk1kVsUodfmIDIB0WgT/BHwAWAzg7i+a2bmRRiU5adOO/Syp387Shiae3bSTrqQzbmQhi+bFqK2Kce6cCsaO1Nx+kYGWVteQu2/p9c0rEU04kkviiSSr3tgZLOLW0MTGHUGpxnkTS/nsuZXUVcU4edp4ze0XiVg6iWBL2D3kZlYIfAmojzYsGa5a9nWwfG0zSxuaWLGumb0dQanGM2aVcd1ZM1g0L8bUCSrVKDKY0kkEnwd+QFCM/i3gvwCND0ha3J2GbXt7CrQ/v2UX7lBRWswHT5xEbVWMs2aXM0qlGkUyJp3/ffPc/ZrUHWZ2FvBkNCFJtmuPJ3h6QwtLGraztL6Jt3e3A3DilLF8qW4OdVUTOe7YMSrVKDJEpJMIfgicksY+yWHbdrf3FGh/Yv0O2uNJRhblc/bscr50wRwWzYsRG6NSjSJD0SETgZmdAZwJVJjZzSlPjSGoQSw5LJl0XnprN0vrt7OkoYlX394DwJTxI7iyZiq11RNZMHOCSjWKZIH+WgRFBPcOFAClKfv3AFdEGZQMTfs6unji9WaW1DexbG0TO/Z1kmdQM30Ct15cRW1VjDmx0ZrbL5JlDpkI3P0x4DEzu9/dNw9iTDKEbG7ZH3b5NLFyYwvxhDOmpIDz58Woqw7m9o9XqUaRrJbOGMEBM/sucBzQ08nr7rWRRSUZ05VIsnpzK8vCFTzXNwWlGmfHRvOps2ZSWxXj1OnjKVCpRpFhI51E8ADwEPAhgqmknwSaowxKBlfr/k4eW9fMkoYmHlvbxJ72LgrzjYWVZVyzICjVOL1MpRpFhqt0EkGZu//UzL6U0l20KurAJDruzutN+3oKtK/Z3ErSoXx0ER847piwVGMFozW3XyQnpPM/PR7+udXMPgi8DUyILiSJQns8wTObdvbM8mlsbQPguGPHcMOi2dRWT+TEyWM1t18kB6WTCL5lZmOBWwjuHxgDfDnSqGRANO1pZ9naJpbUN/HE+h0c6ExQUpjH2bPL+eL5s1lUVcGksSrVKJLrDpsI3P0/w4e7gUXQc2exDDHJZFCqcUlDsILnS427ATh2bAl/fspk6qomcsasMs3tF5GD9HdDWT7wMYI1hv7k7q+Y2YeAvwZGACcPTojSnwOdXTzx+o6eKZ5Nezswg1OmjedrH5hHbVWMqmNKNbdfRA6pvxbBT4GpwLPAXWb2NlAD3Oruvx2M4KRvW3Ye6OnyeXpjC51dSUqLCzh3XgV1VTHOm1tB2WiVahSR9PSXCGqAE909aWYlwDZglru3DE5o0q0rkeT5LbuCO3obmli7PSjVWFk+ik8snE5tdYzTZkygUHP7ReQ96C8RdLp7EsDd281s45EmATO7iGAJ63zgJ+5+Zx/HfAy4HXDgRXf/+JGcY7jafSDOY683s7R+O8vXNbPrQJyCPOP0mRP4Rk01tVUxKitGZzpMERkG+ksEVWb2UvjYgFnhtgHu7if298bhGMPdwIVAI7DKzBa7+2spx8wBvg6c5e6tZhY7imvJau7Ohub9PQXaV29uJZF0JowqorYqRl3VRM6ZW86YEpVqFJGB1V8iqD7K9z4dWO/uGwHM7EHgMuC1lGM+C9zt7q0A7t50lOfMKp1dSZ7dtLNnls/mlgMAVB1TyufPq6S2aiLvmzpOpRpFJFL9LTp3tAvNTQa2pGw3Agt6HTMXwMyeJOg+ut3d/9T7jczseuB6gGnTph1lWJm1Y18Hy8IZPo+/voN9HV0UFeRx1qwyPnNOJbVVMSaP09x+ERk8mV5DoACYA5wPTAFWmNkJ7r4r9SB3vxe4F6CmpsYHO8ij4e68tnUPS+uDRdxebAxKNU4cU8yHTzqWuqoYZ84uY2RRpn8VIpKrovz0eYtg+mm3KeG+VI3AM+4eBzaZ2TqCxJDVaxm1dSZ4asMOljQ0sbS+iW17glKNJ00dx1cumEttVYzjjh2juf0iMiSklQjMbAQwzd3XHsF7rwLmmNlMggRwFdB7RtBvgauB/2dm5QRdRRuP4BxDxtu72npu6npy/Q46upKMKsrn3LkVLKqKsWhejIpSze0XkaHnsInAzD4M/CNBxbKZZvY+4A53v7S/17l7l5ndADxK0P9/n7u/amZ3AKvdfXH43PvN7DUgAXwtW+5TSCSdF7bsYmnDdpY2NFO/NSjVOG3CSD6+YBp1VRM5beZ4igu0nIOIDG3m3n+Xu5mtAWqB5e5+crjvZXc/YRDie5eamhpfvXp1Jk7NnvY4j6/bwZKG7Sxf28zO/Z3k5xk108dTVx2jtmoisypGqctHRIYcM1vj7jV9PZfWMtTuvrvXh1tWDdgejU079rOkPpje+eymnXQlnbEjClk0r4La6omcN6eCsSM1t19Eslc6ieBVM/s4kB/eAHYT8FS0YWVOPJFk1Rs7WVof9Pdv3LEfgLkTR/OZcyqpq45x8tRxKtUoIsNGOongRuA2oAP4BUG//reiDGowrNncysqNLSysLGNm+SiWrw2md65Y28zeji6K8vNYOKuMT545g9qqGFMnjMx0yCIikUgnEVS5+20EyWBYWLO5lWt+vJKOriQYdA+TVJQWc8kJk6itjnH27HJGqVSjiOSAdD7pvmdmxwAPAw+5+ysRxxS5lRtb6OhKBgMdDmfOKuPWi6s4/liVahSR3HPYjm53X0RQmawZ+JGZvWxm34g8sggtrCyjID/4wC8uyOOW98/jxCnjlAREJCelNeLp7tvc/S7g88ALwDcjjSpip04fz6fOmgnA3R8/hVOnj89wRCIimXPYRGBm1WZ2u5m9TFC8/imC5SKyWvddvqdXTshwJCIimZXOGMF9wEPAB9z97YjjGTTt8QQAJbrzV0Ry3GETgbufMRiBDLb2eJL8PKMwX+MCIpLbDpkIzOxX7v6xsEso9U7itCqUDXVt8QQjCvO1HISI5Lz+WgRfCv/80GAEMtja4glKCtUtJCJyyMFid98aPvyiu29O/QG+ODjhRae9M8GIIi0TISKSzifhhX3su3igAxls7V0JDRSLiND/GMEXCL75V5rZSylPlQJPRh1Y1No6E4woUiIQEelvjOAXwB+BvwduTdm/1913RhrVINAYgYhIoL9E4O7+hpn9Ve8nzGxCtieDtniScSNUR0BE5HAtgg8Bawimj6bOs3SgMsK4ItcRT1AyRjWERUQOmQjc/UPhnzMHL5zB030fgYhIrktnraGzzGxU+PhaM/u+mU2LPrRoabBYRCSQzvTRfwEOmNlJwC3ABuDfIo1qEGiwWEQkkE4i6HJ3By4D/tnd7yaYQprVOuJJJQIREdJbfXSvmX0d+AvgHDPLA7J6uk1XIklnIqkxAhER0msRXElQuP5T7r6NoBbBdyONKmLtXUkAJQIREdIrVbkNeAAYa2YfAtrd/V8jjyxCbZ1hLQINFouIpDVr6GPAs8BHgY8Bz5jZFVEHFqV3itJo0TkRkXTGCG4DTnP3JgAzqwD+B3g4ysCi1J0INH1URCS9MYK87iQQaknzdUNWW3ci0BiBiEhaLYI/mdmjwC/D7SuBR6ILKXrdYwRKBCIi6dUs/pqZ/TlwdrjrXnf/TbRhRau7RVCsRCAi0m89gjnAPwKzgJeBr7r7W4MVWJTa45o+KiLSrb++/vuA/wQ+QrAC6Q+P9M3N7CIzW2tm683s1n6O+4iZuZnVHOk53gsNFouIvKO/rqFSd/9x+HitmT13JG9sZvnA3QSlLhuBVWa22N1f63VcKfAl4Jkjef+jocFiEZF39JcISszsZN6pQzAiddvdD5cYTgfWu/tGADN7kGC9otd6Hfe3wHeArx1h7O9Zzw1lhVk9+UlEZED0lwi2At9P2d6Wsu1A7WHeezKwJWW7EViQeoCZnQJMdfc/mNkhE4GZXQ9cDzBt2tGvgN3e1Z0I1CIQEemvMM2iKE8cLl73feC6wx3r7vcC9wLU1NT40Z67vTOBGRTrzmIRkUhvDHsLmJqyPSXc160UOB5YbmZvAAuBxYMxYNxdnczMDn+wiMgwF2UiWAXMMbOZZlYEXAUs7n7S3Xe7e7m7z3D3GcBK4FJ3Xx1hTICK0oiIpIosEbh7F3AD8ChQD/zK3V81szvM7NKozpuO9rhqEYiIdDvsncUW9J9cA1S6+x1hveJj3P3Zw73W3R+h13IU7v7NQxx7floRD4CgRaDxARERSK9F8H+BM4Crw+29BPcHZK12Fa4XEemRzqJzC9z9FDN7HsDdW8M+/6zVFk9QUqBEICIC6bUI4uFdwg499QiSkUYVsfa4WgQiIt3SSQR3Ab8BYmb2beAJ4O8ijSpibfGkZg2JiITSWYb6ATNbA9QRLC9xubvXRx5ZhNrD+whERCS9WUPTgAPA71P3ufubUQYWpbZOzRoSEemWzmDxHwjGBwwoAWYCa4HjIowrUu1dahGIiHRLp2vohNTtcKG4L0YW0SBo60xQosFiERHgPdxZHC4/veCwBw5RyaTT0aU7i0VEuqUzRnBzymYecArwdmQRRUxLUIuIHCydMYLSlMddBGMG/x5NONFTvWIRkYP1mwjCG8lK3f2rgxRP5FSmUkTkYIccIzCzAndPAGcNYjyR6ylTqcFiERGg/xbBswTjAS+Y2WLg18D+7ifd/T8iji0S7WGLoETVyUREgPTGCEqAFoIaxd33EziQ1YlAaw2JiAT6SwSxcMbQK7yTALoddd3gTNEYgYjIwfpLBPnAaA5OAN2yNxF0avqoiEiq/hLBVne/Y9AiGSTdLQIlAhGRQH8jpn21BLKexghERA7WXyKoG7QoBpFuKBMROdghE4G77xzMQAaLBotFRA6Wc5PpuweLi3UfgYgIkIOJoD2eoLggj7y8YTkEIiJyxHIyEWigWETkHTmXCNpUr1hE5CA5mAiSuodARCRF7iWCzoQSgYhIipxLBB1dCUYU5txli4gcUs59IrZ1arBYRCRV7iWCeIKSAiUCEZFukSYCM7vIzNaa2Xozu7WP5282s9fM7CUzW2Jm06OMB8JEoBaBiEiPyBJBWO/4buBiYD5wtZnN73XY80CNu58IPAz8Q1TxdOuIJzV9VEQkRZQtgtOB9e6+0d07gQeBy1IPcPdl7n4g3FwJTIkwHkD3EYiI9BZlIpgMbEnZbgz3HcqngT/29YSZXW9mq81sdXNz81EFFUwfzbmhERGRQxoSn4hmdi1QA3y3r+fd/V53r3H3moqKivd8HndXi0BEpJd0ite/V28BU1O2p4T7DmJmFwC3Aee5e0eE8dDRFdQi0GCxiMg7omwRrALmmNlMMysCrgIWpx5gZicDPwIudfemCGMBUqqTqUUgItIjskTg7l3ADcCjQD3wK3d/1czuMLNLw8O+C4wGfm1mL5jZ4kO83YBQvWIRkXeLsmsId38EeKTXvm+mPL4gyvP31l2URi0CEZF3DInB4sHSXa9YLQIRkXfkVCLoqVeswWIRkR45lQi6B4tLVK9YRKRHTn0i9owRqEUgItIjtxKBpo+KiLxLTiWCdk0fFRF5FyUCEZEcl1OJQLOGRETeLbcSQWd4H4FmDYmI9MipT8T2rgRF+XkU5OfUZYuI9CunPhHbOhMUqxaBiMhBcupTsV21CERE3iWnEkFbPKGBYhGRXnIqEahFICLybjmVCNriSYqVCEREDpJTiaC9M8EIDRaLiBwkpz4VVbheROTdcioRtGuwWETkXXIqEbTFE5QUKBGIiKTKqUTQHk9QohaBiMhBcioRtHVqjEBEpLecSQTuTntXUolARKSXnEkE8YSTSDolmj4qInKQnPlUbFNRGhGRPuVMImhXURoRkT7lXiJQi0BE5CA5kwjUNSQi0rfcSQSdahGIiPQldxKBWgQiIn3KmUTQEQ8K12uwWETkYDmTCN5pEeTMJYuIpCXST0Uzu8jM1prZejO7tY/ni83sofD5Z8xsRlSxaIxARKRvkSUCM8sH7gYuBgQmPZcAAAlNSURBVOYDV5vZ/F6HfRpodffZwD8B34kqnrXb9wLw+vZ9UZ1CRCQrRdkiOB1Y7+4b3b0TeBC4rNcxlwE/Cx8/DNSZmQ10IGs2t3LfE5sAuOEXz7Fmc+tAn0JEJGtFmQgmA1tSthvDfX0e4+5dwG6grPcbmdn1ZrbazFY3NzcfcSArN7aQSDoA8USSlRtbjvg9RESGq6wYOXX3e929xt1rKioqjvj1CyvLKC7MI9+gsCCPhZXvyjUiIjmrIML3fguYmrI9JdzX1zGNZlYAjAUG/Ov6qdPH88BnFrJyYwsLK8s4dfr4gT6FiEjWijIRrALmmNlMgg/8q4CP9zpmMfBJ4GngCmCpu3sUwZw6fbwSgIhIHyJLBO7eZWY3AI8C+cB97v6qmd0BrHb3xcBPgX8zs/XAToJkISIigyjKFgHu/gjwSK9930x53A58NMoYRESkf1kxWCwiItFRIhARyXFKBCIiOU6JQEQkx1lEszUjY2bNwOb3+PJyYMcAhpMNdM25QdecG47mmqe7e5935GZdIjgaZrba3WsyHcdg0jXnBl1zbojqmtU1JCKS45QIRERyXK4lgnszHUAG6Jpzg645N0RyzTk1RiAiIu+Way0CERHpRYlARCTHDctEYGYXmdlaM1tvZrf28XyxmT0UPv+Mmc0Y/CgHVhrXfLOZvWZmL5nZEjObnok4B9LhrjnluI+YmZtZ1k81TOeazexj4e/6VTP7xWDHONDS+Lc9zcyWmdnz4b/vSzIR50Axs/vMrMnMXjnE82Zmd4V/Hy+Z2SlHfVJ3H1Y/BEtebwAqgSLgRWB+r2O+CNwTPr4KeCjTcQ/CNS8CRoaPv5AL1xweVwqsAFYCNZmOexB+z3OA54Hx4XYs03EPwjXfC3whfDwfeCPTcR/lNZ8LnAK8cojnLwH+CBiwEHjmaM85HFsEpwPr3X2ju3cCDwKX9TrmMuBn4eOHgTozs0GMcaAd9prdfZm7Hwg3VxJUjMtm6fyeAf4W+A7QPpjBRSSda/4scLe7twK4e9MgxzjQ0rlmB8aEj8cCbw9ifAPO3VcQ1Gc5lMuAf/XASmCcmU06mnMOx0QwGdiSst0Y7uvzGHfvAnYD2VzIOJ1rTvVpgm8U2eyw1xw2mae6+x8GM7AIpfN7ngvMNbMnzWylmV00aNFFI51rvh241swaCeqf3Dg4oWXMkf5/P6xIC9PI0GNm1wI1wHmZjiVKZpYHfB+4LsOhDLYCgu6h8wlafSvM7AR335XRqKJ1NXC/u3/PzM4gqHp4vLsnMx1YthiOLYK3gKkp21PCfX0eY2YFBM3JlkGJLhrpXDNmdgFwG3Cpu3cMUmxROdw1lwLHA8vN7A2CvtTFWT5gnM7vuRFY7O5xd98ErCNIDNkqnWv+NPArAHd/GighWJxtuErr//uRGI6JYBUwx8xmmlkRwWDw4l7HLAY+GT6+Aljq4ShMljrsNZvZycCPCJJAtvcbw2Gu2d13u3u5u89w9xkE4yKXuvvqzIQ7INL5t/1bgtYAZlZO0FW0cTCDHGDpXPObQB2AmVUTJILmQY1ycC0GPhHOHloI7Hb3rUfzhsOua8jdu8zsBuBRghkH97n7q2Z2B7Da3RcDPyVoPq4nGJS5KnMRH700r/m7wGjg1+G4+JvufmnGgj5KaV7zsJLmNT8KvN/MXgMSwNfcPWtbu2le8y3Aj83sKwQDx9dl8xc7M/slQTIvD8c9/gYoBHD3ewjGQS4B1gMHgL886nNm8d+XiIgMgOHYNSQiIkdAiUBEJMcpEYiI5DglAhGRHKdEICKS45QIZEgys4SZvZDyM6OfY/cNwPnuN7NN4bmeC+9QPdL3+ImZzQ8f/3Wv55462hjD9+n+e3nFzH5vZuMOc/z7sn01Tomepo/KkGRm+9x99EAf28973A/8p7s/bGbvB/7R3U88ivc76pgO975m9jNgnbt/u5/jryNYdfWGgY5Fhg+1CCQrmNnosI7Cc2b2spm9a6VRM5tkZitSvjGfE+5/v5k9Hb7212Z2uA/oFcDs8LU3h+/1ipl9Odw3ysz+YGYvhvuvDPcvN7MaM7sTGBHG8UD43L7wzwfN7IMpMd9vZleYWb6ZfdfMVoVrzH8ujb+WpwkXGzOz08NrfN7MnjKzeeGduHcAV4axXBnGfp+ZPRse29eKrZJrMr32tn7009cPwV2xL4Q/vyG4C35M+Fw5wV2V3S3afeGftwC3hY/zCdYbKif4YB8V7v/fwDf7ON/9wBXh448CzwCnAi8Dowjuyn4VOBn4CPDjlNeODf9cTljzoDumlGO6Y/wz4Gfh4yKCVSRHANcD3wj3FwOrgZl9xLkv5fp+DVwUbo8BCsLHFwD/Hj6+DvjnlNf/HXBt+HgcwVpEozL9+9ZPZn+G3RITMmy0ufv7ujfMrBD4OzM7F0gSfBOeCGxLec0q4L7w2N+6+wtmdh5BsZInw6U1igi+Sfflu2b2DYJ1aj5NsH7Nb9x9fxjDfwDnAH8Cvmdm3yHoTnr8CK7rj8APzKwYuAhY4e5tYXfUiWZ2RXjcWILF4jb1ev0IM3shvP564L9Tjv+Zmc0hWGah8BDnfz9wqZl9NdwuAaaF7yU5SolAssU1QAVwqrvHLVhRtCT1AHdfESaKDwL3m9n3gVbgv9396jTO8TV3f7h7w8zq+jrI3ddZUOvgEuBbZrbE3e9I5yLcvd3MlgMfAK4kKLQCQbWpG9390cO8RZu7v8/MRhKsv/NXwF0EBXiWufufhQPryw/xegM+4u5r04lXcoPGCCRbjAWawiSwCHhXzWUL6jBvd/cfAz8hKPe3EjjLzLr7/EeZ2dw0z/k4cLmZjTSzUQTdOo+b2bHAAXf/OcFifn3VjI2HLZO+PESwUFh36wKCD/UvdL/GzOaG5+yTB9XmbgJusXeWUu9eivi6lEP3EnSRdXsUuNHC5pEFq9JKjlMikGzxAFBjZi8DnwAa+jjmfOBFM3ue4Nv2D9y9meCD8Zdm9hJBt1BVOid09+cIxg6eJRgz+Im7Pw+cADwbdtH8DfCtPl5+L/BS92BxL/9FUBjofzwovwhB4noNeM6CouU/4jAt9jCWlwgKs/wD8Pfhtae+bhkwv3uwmKDlUBjG9mq4LTlO00dFRHKcWgQiIjlOiUBEJMcpEYiI5DglAhGRHKdEICKS45QIRERynBKBiEiO+//YAyD2o3Ji5gAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P_lRQVVlqbNB",
        "colab_type": "text"
      },
      "source": [
        "По сравнению с бейзлайном на всех данных precision меток 0 и 1 снизился до 0.95 0.82, recall метки 1 снизился до 0.54. Результаты по сравнению с top 30000 features улучшились."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SRnXWT4Y0zFQ",
        "colab_type": "text"
      },
      "source": [
        "## 3. Ансамбль из моделей в sklearn (ансамблевые модели типа randomforest не считаются). Нужно минимум 5 разных моделей - 2 балла"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aYCAMTZMT3zX",
        "colab_type": "text"
      },
      "source": [
        "На даунсэмпле"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IKf6Qw7VX-jq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = data_downsampled['comment_text']\n",
        "y = data_downsampled['toxic']"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SFeZJptiYRvV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_QzNYUzXuWZw",
        "colab_type": "text"
      },
      "source": [
        "Без параметров"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gTFKxISXwz6Y",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 872
        },
        "outputId": "7ea914f9-1c69-4240-9c6f-dc176b675a9f"
      },
      "source": [
        "from sklearn.utils.testing import all_estimators\n",
        "\n",
        "estimators = all_estimators()\n",
        "\n",
        "for name, class_ in estimators:\n",
        "    if hasattr(class_, 'predict_proba'):\n",
        "        print(name)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "AdaBoostClassifier\n",
            "BaggingClassifier\n",
            "BayesianGaussianMixture\n",
            "BernoulliNB\n",
            "CalibratedClassifierCV\n",
            "CategoricalNB\n",
            "ClassifierChain\n",
            "ComplementNB\n",
            "DecisionTreeClassifier\n",
            "DummyClassifier\n",
            "ExtraTreeClassifier\n",
            "ExtraTreesClassifier\n",
            "GaussianMixture\n",
            "GaussianNB\n",
            "GaussianProcessClassifier\n",
            "GradientBoostingClassifier\n",
            "GridSearchCV\n",
            "HistGradientBoostingClassifier\n",
            "KNeighborsClassifier\n",
            "LabelPropagation\n",
            "LabelSpreading\n",
            "LinearDiscriminantAnalysis\n",
            "LogisticRegression\n",
            "LogisticRegressionCV\n",
            "MLPClassifier\n",
            "MultiOutputClassifier\n",
            "MultinomialNB\n",
            "NoSampleWeightWrapper\n",
            "NuSVC\n",
            "OneVsRestClassifier\n",
            "Pipeline\n",
            "QuadraticDiscriminantAnalysis\n",
            "RFE\n",
            "RFECV\n",
            "RadiusNeighborsClassifier\n",
            "RandomForestClassifier\n",
            "RandomizedSearchCV\n",
            "SGDClassifier\n",
            "SVC\n",
            "StackingClassifier\n",
            "VotingClassifier\n",
            "_BinaryGaussianProcessClassifierLaplace\n",
            "_ConstantPredictor\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.utils.testing module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.utils. Anything that cannot be imported from sklearn.utils is now part of the private API.\n",
            "  warnings.warn(message, FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/externals/joblib/__init__.py:15: FutureWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "57J7m_IZQwH-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "clf1 = MultinomialNB()\n",
        "clf2 = SVC()\n",
        "clf3 = LogisticRegression()\n",
        "clf4 = DecisionTreeClassifier()\n",
        "clf5 = XGBClassifier()\n",
        "\n",
        "eclf = VotingClassifier(estimators=[\n",
        "        ('MNB', clf1), ('SVC', clf2), ('LogReg', clf3),  ('DecTree', clf4),  ('XGB', clf5)], voting='hard')"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7dtvN7UXY1B9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "voting = Pipeline([\n",
        "    ('vect', CountVectorizer(analyzer='word', max_features=200)),\n",
        "    ('tfidf', TfidfTransformer(sublinear_tf=True)),\n",
        "    ('to_dense', FunctionTransformer(lambda x: x.todense(), accept_sparse=True)), \n",
        "    ('clf', eclf),\n",
        "    ])"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tWRuK7IVFIrs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "voting = voting.fit(X_train, y_train)"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FWCocf56aks3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "predictions = voting.predict(X_test)"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vY_LCGr0Giws",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "61fced32-080d-444f-9a46-931cde7b203f"
      },
      "source": [
        "print(\"Precision: {0:6.2f}\".format(precision_score(y_test, predictions, average='macro')))\n",
        "print(\"Recall: {0:6.2f}\".format(recall_score(y_test, predictions, average='macro')))\n",
        "print(\"F1-measure: {0:6.2f}\".format(f1_score(y_test, predictions, average='macro')))\n",
        "print(\"Accuracy: {0:6.2f}\".format(accuracy_score(y_test, predictions)))"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Precision:   0.83\n",
            "Recall:   0.83\n",
            "F1-measure:   0.83\n",
            "Accuracy:   0.83\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OFWy0ZAht2xX",
        "colab_type": "text"
      },
      "source": [
        "По сравнению с бейзлайном на даунсэмпле метрики получились на порядок меньше. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iwW1oYe1JjkH",
        "colab_type": "text"
      },
      "source": [
        "## 4. Любая нейронная модель (минимум 5 слоев) с Dropout, Pooling и колбеками - 2 балла"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p1OScZyPUtJW",
        "colab_type": "text"
      },
      "source": [
        "На даунсэмпле"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qG0E6An4JzOH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def preprocess(text):\n",
        "    tokens = text.lower().split()\n",
        "    tokens = [token.strip(punctuation) for token in tokens]\n",
        "    return tokens"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OwfSBM3_u8j9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras import backend as K\n",
        "def f1(y_true, y_pred):\n",
        "    def recall(y_true, y_pred):\n",
        "        \"\"\"Recall metric.\n",
        "\n",
        "        Only computes a batch-wise average of recall.\n",
        "\n",
        "        Computes the recall, a metric for multi-label classification of\n",
        "        how many relevant items are selected.\n",
        "        \"\"\"\n",
        "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
        "        recall = true_positives / (possible_positives + K.epsilon())\n",
        "        return recall\n",
        "\n",
        "    def precision(y_true, y_pred):\n",
        "        \"\"\"Precision metric.\n",
        "\n",
        "        Only computes a batch-wise average of precision.\n",
        "\n",
        "        Computes the precision, a metric for multi-label classification of\n",
        "        how many selected items are relevant.\n",
        "        \"\"\"\n",
        "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
        "        precision = true_positives / (predicted_positives + K.epsilon())\n",
        "        return precision\n",
        "    precision = precision(y_true, y_pred)\n",
        "    recall = recall(y_true, y_pred)\n",
        "    return 2*((precision*recall)/(precision+recall+K.epsilon()))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "12WhPnO3u_Wq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vocab = Counter()\n",
        "\n",
        "for text in data_downsampled['comment_text']:\n",
        "    vocab.update(preprocess(text))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-02akU7evOiB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "858c4a69-3e3a-4e4f-e4fd-a29e798a5a96"
      },
      "source": [
        "len(vocab)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "102673"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 131
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TPhA5DtmvXeR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "filtered_vocab = set()\n",
        "\n",
        "for word in vocab:\n",
        "    if vocab[word] > 2:\n",
        "        filtered_vocab.add(word)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BqAszN97va86",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "39682a0d-65a8-4ad2-c2c2-9234f6c0daa1"
      },
      "source": [
        "len(filtered_vocab)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "27207"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 133
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A1Mto0XEvcxR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "word2id = {'UNK':1, 'PAD':0}\n",
        "\n",
        "for word in filtered_vocab:\n",
        "    word2id[word] = len(word2id)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CSl1OsRbvqsB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "id2word = {i:word for word, i in word2id.items()}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AWrORXoGvtKR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = []\n",
        "\n",
        "for text in data_downsampled['comment_text']:\n",
        "    tokens = preprocess(text)\n",
        "    ids = [word2id.get(token, 1) for token in tokens]\n",
        "    X.append(ids)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8WVEb7dav09x",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "MAX_LEN = max(len(x) for x in X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DJTdvavYv5JP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "MEAN_LEN = np.median([len(x) for x in X])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fHWCjdfawGVY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "1f70755d-bba3-40f4-9660-77a028daac95"
      },
      "source": [
        "MAX_LEN, MEAN_LEN"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2321, 28.0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 139
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ElBhVeXNwNL_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "MAX_LEN = 35"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZCq-yXZ6weDX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = tf.keras.preprocessing.sequence.pad_sequences(X, maxlen=MAX_LEN)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pAY_p3KnwhD3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "51fa2295-d522-4b00-dc6c-a86c597fcf34"
      },
      "source": [
        "X.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(42768, 35)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 142
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aWo-T9Vww0rq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = data_downsampled.toxic.values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d0ZCLU6pxBSe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.05, stratify=y)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8eGRVart3enH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "filepath = '/gdrive/My Drive/exam/model.weights'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3L00x3q71RJ9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath, \n",
        "                                                monitor='val_f1', \n",
        "                                                verbose=1, \n",
        "                                                save_weights_only=True, \n",
        "                                                save_best_only=True,\n",
        "                                                mode='max', \n",
        "                                                save_freq='epoch' \n",
        "                                               )\n",
        "\n",
        "early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_f1', \n",
        "                                              min_delta=0.01, \n",
        "                                              patience=3, \n",
        "                                              verbose=1, \n",
        "                                              mode='max',\n",
        "                                              )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VX8TR6ngzjwz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "inputs = tf.keras.layers.Input(shape=(MAX_LEN,))\n",
        "embeddings = tf.keras.layers.Embedding(input_dim=len(word2id), output_dim=100)(inputs, )\n",
        "convs = []\n",
        "\n",
        "conv1 = tf.keras.layers.Conv1D(kernel_size=2, filters=24, strides=1)(embeddings)\n",
        "drop1 = tf.keras.layers.Dropout(0.1)(conv1)\n",
        "conv2 = tf.keras.layers.Conv1D(kernel_size=1, filters=16, strides=1, activation='relu')(drop1)\n",
        "pool1 = tf.keras.layers.GlobalAveragePooling1D()(conv2)\n",
        "drop2 = tf.keras.layers.Dropout(0.2)(pool1)\n",
        "convs.append(drop2)\n",
        "\n",
        "conv3 = tf.keras.layers.Conv1D(kernel_size=4, filters=32, strides=1)(embeddings)\n",
        "conv4 = tf.keras.layers.Conv1D(kernel_size=6, filters=16, strides=1, activation='relu')(conv3)\n",
        "pool2 = tf.keras.layers.GlobalMaxPooling1D()(conv4)\n",
        "drop3 = tf.keras.layers.Dropout(0.3)(pool2)\n",
        "convs.append(drop3)\n",
        "\n",
        "conv5 = tf.keras.layers.Conv1D(kernel_size=8, filters=32, strides=1, kernel_regularizer='l2')(embeddings)\n",
        "conv6 = tf.keras.layers.Conv1D(kernel_size=10, filters=16, strides=1, activation='relu')(conv5)\n",
        "pool3 = tf.keras.layers.GlobalAveragePooling1D()(conv6)\n",
        "drop4 = tf.keras.layers.Dropout(0.4)(pool3)\n",
        "convs.append(drop4)\n",
        "\n",
        "conv7 = tf.keras.layers.Conv1D(kernel_size=2, filters=16, strides=2, kernel_regularizer='l2')(embeddings)\n",
        "conv8 = tf.keras.layers.Conv1D(kernel_size=2, filters=16, strides=2, activation='relu')(conv7)\n",
        "pool4 = tf.keras.layers.GlobalMaxPooling1D()(conv8)\n",
        "drop5 = tf.keras.layers.Dropout(0.5)(pool4)\n",
        "convs.append(drop5)\n",
        "\n",
        "concat = tf.keras.layers.concatenate(convs)\n",
        "\n",
        "dense = tf.keras.layers.Dense(64, activation='relu')(concat)\n",
        "\n",
        "outputs = tf.keras.layers.Dense(1, activation='sigmoid')(dense)\n",
        "\n",
        "model = tf.keras.Model(inputs=inputs, outputs=outputs)\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
        "model.compile(optimizer=optimizer,\n",
        "              loss='binary_crossentropy',\n",
        "              metrics=[f1])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nks7xmreypWL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 459
        },
        "outputId": "4b40ff7b-f390-401e-fcff-0726d712d729"
      },
      "source": [
        "model.fit(X_train, y_train, \n",
        "          validation_data=(X_valid, y_valid),\n",
        "          batch_size=2000,\n",
        "         epochs=10,\n",
        "         callbacks=[checkpoint, early_stop])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "21/21 [==============================] - ETA: 0s - loss: 1.2324 - f1: 0.4832\n",
            "Epoch 00001: val_f1 improved from -inf to 0.61628, saving model to /gdrive/My Drive/exam/model.weights\n",
            "21/21 [==============================] - 16s 754ms/step - loss: 1.2324 - f1: 0.4832 - val_loss: 0.9908 - val_f1: 0.6163\n",
            "Epoch 2/10\n",
            "21/21 [==============================] - ETA: 0s - loss: 0.8170 - f1: 0.7328\n",
            "Epoch 00002: val_f1 improved from 0.61628 to 0.82849, saving model to /gdrive/My Drive/exam/model.weights\n",
            "21/21 [==============================] - 15s 736ms/step - loss: 0.8170 - f1: 0.7328 - val_loss: 0.5873 - val_f1: 0.8285\n",
            "Epoch 3/10\n",
            "21/21 [==============================] - ETA: 0s - loss: 0.4961 - f1: 0.8464\n",
            "Epoch 00003: val_f1 improved from 0.82849 to 0.87461, saving model to /gdrive/My Drive/exam/model.weights\n",
            "21/21 [==============================] - 15s 733ms/step - loss: 0.4961 - f1: 0.8464 - val_loss: 0.4069 - val_f1: 0.8746\n",
            "Epoch 4/10\n",
            "21/21 [==============================] - ETA: 0s - loss: 0.3301 - f1: 0.9002\n",
            "Epoch 00004: val_f1 improved from 0.87461 to 0.87953, saving model to /gdrive/My Drive/exam/model.weights\n",
            "21/21 [==============================] - 16s 740ms/step - loss: 0.3301 - f1: 0.9002 - val_loss: 0.3302 - val_f1: 0.8795\n",
            "Epoch 5/10\n",
            "21/21 [==============================] - ETA: 0s - loss: 0.2389 - f1: 0.9263\n",
            "Epoch 00005: val_f1 did not improve from 0.87953\n",
            "21/21 [==============================] - 15s 732ms/step - loss: 0.2389 - f1: 0.9263 - val_loss: 0.3154 - val_f1: 0.8753\n",
            "Epoch 6/10\n",
            "21/21 [==============================] - ETA: 0s - loss: 0.1904 - f1: 0.9389\n",
            "Epoch 00006: val_f1 improved from 0.87953 to 0.88432, saving model to /gdrive/My Drive/exam/model.weights\n",
            "21/21 [==============================] - 15s 737ms/step - loss: 0.1904 - f1: 0.9389 - val_loss: 0.3319 - val_f1: 0.8843\n",
            "Epoch 00006: early stopping\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f2c2f534f28>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IQM_yeqSyrqj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        },
        "outputId": "bd9ebda9-a83c-49f9-c459-b4987ce410be"
      },
      "source": [
        "print(model.history.history.keys())\n",
        "# summarize history for accuracy\n",
        "plt.plot(model.history.history['f1'])\n",
        "plt.plot(model.history.history['val_f1'])\n",
        "plt.title('model f1')\n",
        "plt.ylabel('f1')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'val'], loc='upper left')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dict_keys(['loss', 'f1', 'val_loss', 'val_f1'])\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xV9f3H8dcnO2QwkrASMGEIAQdTUax7MBSsC7Xa2voT+3NrtdVqrbXa+uuyS+tobat1FHChoLhAq0IlUYYk7GESVsJKQnby/f1xL3gJYefek9z7fj4e95F7zj05389F8/2c8z3nfD/mnENERCJXlNcBiIiIt5QIREQinBKBiEiEUyIQEYlwSgQiIhFOiUBEJMIpEYgcITP7h5k9dJDbrjWzs/fxWaKZvWFmO8xsautGKbJvSgQibcclQDcgzTl3qZn1MLPpZrbezJyZZXsbnoQrJQKRtuMoYLlzrsG/3AS8DVzsXUgSCZQIJCL4h2TuMrNFZrbTzP5mZt3M7C0zqzCz98ysc8D2E8xsiZltN7M5ZpYb8NlQM/vc/3v/BhKatXW+mS3w/+6nZnbcQcT3M+B+YJKZVZrZtc65Tc65x4H5rfcvIbI3JQKJJBcD5wBHAxcAbwE/BjLw/S3cAmBmRwMvArf5P5sJvGFmcWYWB7wGPAd0AaYScMRuZkOBZ4DrgTTgSWC6mcXvLzDn3E+BXwD/ds4lO+f+1krfWeSAlAgkkvzJf5RdAvwH+K9z7gvnXA3wKjDUv90kYIZz7l3nXD3wGyAROBkYBcQCv3fO1TvnprHnEftk4Enn3H+dc43OuX8Ctf7fE2mTYrwOQCSENgW8r25hOdn/viewbtcHzrkmMysCMoFGoMTtOVvjuoD3RwHfMbObA9bF+fcp0iYpEYjsbT1w7K4FMzOgF1ACOCDTzCwgGfQGVvnfFwEPO+ceDmG8IkdEQ0Mie5sCjDezs8wsFvgBvuGdT4G5QANwi5nFmtlFwAkBv/s08H0zO9F8ksxsvJmlHE4gZpYA7Lq+EO9fFmlVSgQizTjnlgFXAX8CyvBdWL7AOVfnnKsDLgKuAbbiu57wSsDv5gHXAX8GtgEr/dsermqg0v9+qX9ZpFWZCtOIiEQ2nRGIiEQ4JQIRkQinRCAiEuGUCEREIly7e44gPT3dZWdnex2GiEi7kp+fX+acy2jps3aXCLKzs8nLy/M6DBGRdsXM1u3rMw0NiYhEOCUCEZEIp0QgIhLh2t01gpbU19dTXFxMTU2N16EEVUJCAllZWcTGxnodioiEkbBIBMXFxaSkpJCdnY1vosjw45xjy5YtFBcXk5OT43U4IhJGwmJoqKamhrS0tLBNAgBmRlpaWtif9YhI6IVFIgDCOgnsEgnfUURCLyyGhkREwoVzjp11jZRV1FJW6XuVVtZRVlHLWbldOS6rU6u3qUTQCrZv384LL7zADTfccEi/N27cOF544QU6dWr9/7Ai0nY45yivafB17BW1lFXW7e7kyyprKa3Yc7mmvqnF/WSkxCsRtFXbt2/n8ccf3ysRNDQ0EBOz73/imTNnBjs0EQkS5xzbq+r9R+z+zj3gKH53Z19RS9nOOuoa9u7cowy6JMWTnhxHRko8OelJpCfHkZ4c73ul+D9LjqdLUhwx0cEZzVciaAV33303q1atYsiQIcTGxpKQkEDnzp1ZunQpy5cv58ILL6SoqIiamhpuvfVWJk+eDHw9XUZlZSVjx47llFNO4dNPPyUzM5PXX3+dxMREj7+ZSGRpbHJsq9rVgdcFDM3suVxWWcuWyjoamvYu7BUTZaQFdOb9u6aQnuLrzL/u4H2fd+4QR3SU99f+wi4R/OyNJRSsL2/VfQ7qmcpPLxi8z88feeQRvvzySxYsWMCcOXMYP348X3755e7bPJ955hm6dOlCdXU1I0eO5OKLLyYtLW2PfaxYsYIXX3yRp59+mssuu4yXX36Zq666qlW/h0gkamhsYuvOuv0etZf6h2u27qylhb6duOgo35F6SjzdUhMY3DO1xaP29OR4OibGEtUGOvdDEXaJoC044YQT9rjX/49//COvvvoqAEVFRaxYsWKvRJCTk8OQIUMAGD58OGvXrg1ZvCLtTX1j0x5H7aW7OvZmR+1llXVsq6qjpYq8CbFRuzvzrM4dGNq709ede3L87o4/PTme1ISYsL5rL+wSwf6O3EMlKSlp9/s5c+bw3nvvMXfuXDp06MDpp5/e4rMA8fHxu99HR0dTXa0a5SIAW3fWUbihnMIN5RSsL6dgQzkrN1e2OCyTFBe9u/POSU9iZHaX3UftGc3G3pPiosO6cz8UYZcIvJCSkkJFRUWLn+3YsYPOnTvToUMHli5dyrx580IcnUj70NTkWLe1yt/Z76BwQwUF68vZWP71gVO31Hhye6RyxsCu9OrcYfdR+65hmcS4aA+/QfulRNAK0tLSGD16NMcccwyJiYl069Zt92djxozhiSeeIDc3lwEDBjBq1CgPIxVpG6rqGli6sWL3UX7hhnKWbqygqq4RgOgoo19GMif1TSO3RwqDenQkt0cKacnxB9izHA5zLQ2etWEjRoxwzQvTFBYWkpub61FEoRVJ31XaP+ccm8prfR2+/1W4vpw1W3buHrdPSYhhUI9UcnukMqhnKoN6pNKvazIJsTq6b01mlu+cG9HSZzojEJFWUd/YxKrSyt1H+AUbyincUMHWnXW7t+ndpQO5PVKYOCTTd6TfM5XMTokaq/eYEoGIHLId1fV7XLwt3FDOik2V1DX6HpqKi4liYPcUzh3UbfeR/sDuKaQkaAr1tkiJQET2qanJUbytmoINOyjwX7wt3FBOyfav72pLT44jt0cq3z0lm0E9fEM7OelJQXsKVlqfEoGIAFBT38iyXRdw/Uf5hRsqqKxtAHzTIfTJSGb4UZ25atRRDOqZSm6PFLqmJHgcuRwpJQKRCLS5omb37Zm7Ov7VpZW7n6pNjo8ht0cKFw3L3H0hd0D3FF3ADVNBTQRmNgb4AxAN/NU590izz48CngEygK3AVc654mDGJBJJGhqbWFO2c/cdO76Ov4Kyytrd22R2SiS3Ryrjjunuv2unI1mdE9vdNAly+IKWCMwsGngMOAcoBuab2XTnXEHAZr8BnnXO/dPMzgR+CVwdrJjaiuTkZCorK70OQ8JMRU09SzfueZS/bGMFtf5ZL+Oio+jfLZnTB2R8fbtmj1Q6dtAF3EgXzDOCE4CVzrnVAGb2EjARCEwEg4A7/O9nA68FMR6RsNLU5Ji3ZgvT8ovJW7uNr7ZW7f6sc4dYBvVM5dsnHbX7rp2+GcnE6gKutCCYiSATKApYLgZObLbNQuAifMNH3wRSzCzNObclcCMzmwxMBujdu3fQAj5cd999N7169eLGG28E4IEHHiAmJobZs2ezbds26uvreeihh5g4caLHkUo4KNlezcv5xUzNL6JoazUpCTF8o386k0b22n2k3y01Xvfmy0Hz+mLxncCfzewa4COgBGhsvpFz7ingKfA9WbzfPb51N2xc3LpRdj8Wxj6yz48nTZrEbbfdtjsRTJkyhVmzZnHLLbeQmppKWVkZo0aNYsKECfrjlMNS29DIO0s2MSWviI9XluEcjO6Xxp3nDuC8wd11EVeOSDATQQnQK2A5y79uN+fcenxnBJhZMnCxc257EGMKiqFDh7J582bWr19PaWkpnTt3pnv37tx+++189NFHREVFUVJSwqZNm+jevbvX4Uo7smT9DqbmFfPaghK2V9XTs2MCN5/Zn0uHZ9GrSwevw5MwEcxEMB/ob2Y5+BLA5cCVgRuYWTqw1TnXBNyD7w6iI7OfI/dguvTSS5k2bRobN25k0qRJPP/885SWlpKfn09sbCzZ2dktTj8t0tz2qjpeX7CeKXlFLFlfTlx0FOcd053LRmRxct/0NlHRSsJL0BKBc67BzG4CZuG7ffQZ59wSM3sQyHPOTQdOB35pZg7f0NCNwYon2CZNmsR1111HWVkZH374IVOmTKFr167ExsYye/Zs1q1b53WI0oY1NTk+WVXGlLxiZi3ZSF1DE4N7pvKzCYOZOKQnnTrEeR2ihLGgXiNwzs0EZjZbd3/A+2nAtGDGECqDBw+moqKCzMxMevTowbe+9S0uuOACjj32WEaMGMHAgQO9DlHaoKKtVUzNL+bl/GJKtlfTMTGWK0/ozSXDszgms6PX4UmE8PpicVhZvPjri9Tp6enMnTu3xe30DEFkq6lvZNaSjfx7fhGfrtqCGZzSL517xg3k7NxuuvAbCZqaoHYH1OyA6u2+nzU7oGb7/ted/iM45uJWD0eJQCQEnHMsLtnBlLwiXl+wnoqaBnp1SeSOc47m4uFZZHZK9DpEORTOQX31PjrvgE68envA8q7tdkBtObCfGyAtChI6+l+dfD+7DoTEzkH5OkoEIkG0dWcdr35RwtS8IpZurKBDjOOC3M5ccnwaw3smEtVYC9XLobwGGqqh/gA/G2p9HVBDTcDPGjCDmHiIjvf9jEmAmDj/z+brA95HxzVbt2t9C/uJjvO1Ey4aGwI66Oad90EcnTfV73//sUmQ2OnrDj01C7oODlgX8FnzdfEpIf23DptE4JwL+3v021s1uTbPuWYdakDHetidcjWuvobKnRXsrKykoa6KCdRxRVQ9CYn1RLkGWIHvdahiEnyv2MS9fzoH1dt8Me1+1UBjwPvWEN08mRwooTRfn3B4SSgwmUVFf/3fr67y0IZWApfrDjBEGxXzdce8q6Pu1HvfnXdipz079+j2M3VHWCSChIQEtmzZQlpaWtgmA+ccW7ZsISFBU/7u0+alUPA6lC3fRwff7OeRdI5RMRCTCLEJu3/WWhxlNVGsr4QdDfE0xXSkW3onendLo0Nq6r478ZiEPfbT4s+Y+CM7QnQOGuv937t2zwSxR+Ko23vdXtvX7Xs/dTuhakuz/ez6nWpwTYf/HXb/28f6/j3qq8Ht9fzpnuJT9+ycu+Qc+Gh813Jsh/A6A9qPsEgEWVlZFBcXU1pa6nUoQZWQkEBWVpbXYbQdzsGmJb7Ov+B1KFsGGHTO9v0R7+pEO3Q5QCfc0s/4/XfM0b4/naq6Bt5avJF/5xXx2ZqtRBmcdnQGl43oxVm53YiLaSNz+5j5j649vg21saH1klBswt6dd/NOPkoX3g9GWCSC2NhYcnJyvA5DQsE52Ljo685/y0rfhbWjRsMJ10HuBZAS3Ke3nXN88dU2puYV8cbCDVTWNpCd1oG7zhvAxcOy6N5RZ237FB3je8UleR2JBAiLRCBhzjlY//nXnf+2tWDRkPMNOOkmGHg+JGcEPYzSilpe+6KEKXlFrNhcSWJsNOOO7cGkkb0Ymd05bIclJfwpEUjb1NQEJflQ8BoUTIcdX/nG5fucDt/4AQwYD0lpQQ+jobGJOctKmZJXxAdLN9PQ5BjWuxOPXHQs44/roWLsEhaUCKTtaGqCov/6jvoLp0N5ie/Okr5nwhn3wICxQbuPurlVpZVMzSvm5c+LKa2oJT05ju+dksNlI7Lo1zUlJDGIhIoSgXirqRHWferv/N+Ayo2+2wT7nQ1n/RQGjPFd9AuBytoGZi7awJS8IvLWbSM6yjhjQFcuG5HFGQO7qqiLhC0lAgm9xgZY9/HXnf/OUt+dOP3PgUET4ejzfA/UhIBzjrx125gyv4gZizdQVddIn4wk7hk7kG8Oy6Rrii78SvhTIpDQaKyHNR/6O/83oXqr78nLo8/zdf79zwnpnSSby2t4+XPfE7+ry3aSFBfNBcf15LKRWQzrrQu/ElmUCCR4Gmph9Rxf5790hu+JzrgU33DPoInQ9yyIC11xlbqGJj5YupmpeUXMWV5KY5NjZHZn/vf0vow7tgdJ8fpzkMik//OlddXXwKoPfJ3/srd8MyzGd4SB43ydf58zfA8ChdCKTRVMySvilc9L2LKzjq4p8Uw+tQ+XDs+iT0ZySGMRaYuUCOTI1VXByvd8nf/yt31zuCR08j3cNWii75bPED/RWl5Tz5sLfRd+FxRtJybKODu3G5eNzOLU/hnE6MKvyG5KBHJ46nbCinf8nf87UL8TOqT55kofNBFyTvVk0q2C9eX89ePVzFy8gZr6Jvp3Tea+8blcODST9OT4kMcj0h4oEcjBq62A5bN8D3mteM83iVhSBhw/CQZd6JvmIdq7/6Wm5Rfz41cXEx8dxUXDsrhsRC+Oz+qoC78iB6BEIPtXs8M31l/wOqx83zcBWHJ3GHa178i/90meT+zV0NjEwzML+fsnazm5bxp/vnIYXZJU41fkYCkRyN6qtn7d+a/6wFeAIzUTRnwPBl8IWSdAVNsYY9+2s44bX/icT1dt4bujs7l3XK7G/0UOkRKB+OzcAkvf9HX+az6Epgbo2BtOvN437JM5vM10/rsUbijnumfz2FxRy68vOY5LR/TyOiSRdkmJIJJVbvZ1/kteg7Uf+4p8dM72zeg5aCL0HNpmC3PMWLSBO6cuJDUxhinXn8SQXp28Dkmk3VIiiDQVG33TOhS8Dus+8VWM6tIXTrnN1/l3P67Ndv4ATU2O3727nD/PXsnQ3p148qrhdE3VNBAiR0KJIBLsKPHN5lnwOnw1D3CQMRBOvcvX+Xcd1KY7/13Ka+q5/aUFvL90M5NG9OLBCwcTH6MKVCJHSokgnK39BN57AIo/8y13HQxn/BhyJ0DXgZ6GdqhWlVYy+dk81m2p4sGJg7l61FG6LVSklSgRhKudW2Dqd3yzep75E9+Rf3p/r6M6LLOXbuaWF78gNiaK5649kZP6Br8gjUgkUSIIV2//CKq3weTXoPsxXkdzWJxz/OXDVfx61jJyu6fy1LeHk9U5dJPUiUQKJYJwtHQGLJ4Kp9/TbpNAdV0jd01byJuLNnD+cT349SXHkxin6wEiwaBEEG6qtsKbt0O3Y+CUO7yO5rAUba3i+ufyKdxYzo/GDOT7p/XR9QCRIFIiCDezfgw7y+BbU0M+42drmLtqCze+8Dn1jU08852RnDGwq9chiYQ9JYJwsnwWLHzRd1toj+O9juaQOOd4du46HnyzgOy0Djz97RGqFSASIkoE4aJ6O7xxG2Tk+hJBO1Lb0MhPXvuSKXnFnJ3bld9NGkJqQuinsBaJVEoE4eKde6FyE1z+PMS0n3n3N5fXcP2/8vniq+3cfGY/bj/7aKKidD1AJJSUCMLByvfgi3/BKbdD5jCvozloX3y1je//K5/y6gYe/9Ywxh3bw+uQRCJSUKeTNLMxZrbMzFaa2d0tfN7bzGab2RdmtsjMxgUznrBUUw7Tb4X0AXDaXv/EbdbUvCImPTmPuJgoXrnhZCUBEQ8F7YzAzKKBx4BzgGJgvplNd84VBGx2HzDFOfcXMxsEzASygxVTWHr3J1CxHr73TsiLwh+O+sYmHp5RyD8+9RWReezKYXRWERkRTwVzaOgEYKVzbjWAmb0ETAQCE4EDUv3vOwLrgxhP+Fk9B/L/ASffDL1Geh3NAW3dWceNz3/O3NVb+N7oHH48bqCKyIi0AcFMBJlAUcByMXBis20eAN4xs5uBJODslnZkZpOByQC9e/du9UDbpdpKmH4zpPWDM+71OpoDCiwi85tLj+eS4VlehyQifl4fjl0B/MM5lwWMA54zs71ics495Zwb4ZwbkZGREfIg26T3HoDtRTDxMYhN9Dqa/ZqxaAMXPf4p9Y1NTLn+JCUBkTYmmGcEJUBg7cAs/7pA1wJjAJxzc80sAUgHNgcxrvZvzX9g/tMw6gboPcrraPapqcnx23eX8djsVQzr3YknVERGpE0K5hnBfKC/meWYWRxwOTC92TZfAWcBmFkukACUBjGm9q9uJ0y/CTrn+KaXbqPKa+q57tk8Hpu9ikkjevHi5FFKAiJtVNDOCJxzDWZ2EzALiAaecc4tMbMHgTzn3HTgB8DTZnY7vgvH1zjnXLBiCgvv/xy2rYVrZkBc25ySeVVpJdc9m8dXKiIj0i4E9YEy59xMfLeEBq67P+B9ATA6mDGElXVz4b9PwMjrIPsUr6NpUWARmX/9z4mM6qMiMiJtnZ4sbi/qquD1G6FTLzj7Aa+j2YtzjsfnrOI376iIjEh7o0TQXsx+GLaugm9Ph/i2NStnVV0DP5y2SEVkRNopJYL2oOgzmPc4DP8u9DnN62j2ULS1isnP5bNURWRE2i0lgrauvsY3JJSaCec86HU0e5i7ags3PJ9PQ5PjmWtGcsYAFZERaY+UCNq6Ob+EsuVw1SuQkHrg7UPAOcc/P13Lz2cUqoiMSBhQImjLSvLh0z/C0Kuh31leRwPsXUTm0UlDSFERGZF2TYmgrWqohddugOTucN7DXkcDwKbyGr6vIjIiYUeJoK368FdQuhSunAoJHb2Ohi++2sb1z+VTWdvAX741jLGqHyASNpQI2qL1C+DjR+H4K+Hoc72Ohil5Rdz36pd06xjPs9eezMDubeNahYi0DiWCtqahzneXUFIGjPmFp6EEFpEZ3S+NP1+hIjIi4UiJoK35z29h05dwxUuQ2NmzMAKLyFx7Sg73jFURGZFwpUTQlmxcDP/5DRx7GQwY61kYBevLmfycr4jMby89notVP0AkrCkRtBWN9b67hBK7wNj/8yyMGYs2cOfUhaQmxjDl+pMY0quTZ7GISGgoEbQVH/8eNi6Cy56DDl1C3nxjk+N3KiIjEpGUCNqCTQXw4f/B4Itg0ISQN19eU89tLy3gg6WbuXxkL342cTDxMZo0TiRSKBF4rbEBXr/B96zAuF+HvPnAIjI/nziYq1RERiTiKBF4be6fYP0XcMnfISk9pE2riIyIgBKBt0qXwexfQO4EGPzNkDUbWERmUI9UnrxaRWREIpkSgVeaGn0PjsUlw/jfQoiGY6rqGrhr2iJmLNrABcf35FcXH6ciMiIRTonAK/Meh+L5cNFfITk08/gHFpG5e+xArj9VRWRERInAG2Ur4YOHYMA4OPaSkDT56aoybnz+cxqaHH+/ZiSnq4iMiPgpEYTariGhmHg4/9GgDwkFFpHJSU/i6W+PICc9Kahtikj7okQQap89BUXz4MInIKV7UJuqbWjkvle/ZGq+isiIyL4pEYTSllXw3s+g/7lw/OVBb+7eV79kWn4xt5zZj9tUREZE9kGJIFSammD6zRAdBxf8IehDQouKtzMtv5jrT+3DHecOCGpbItK+KRGESt7fYN0nMOHPkNozqE0553hoRiFdkuK48cx+QW1LRNo/TTAfCtvWwrs/hb5nwdCrgt7crCWb+GzNVm4/52hSdU1ARA5AiSDYnPMNCVlUSIaE6hqaeOStQvp1TeaKkb2C2paIhAclgmDL/zus+QjO/Tl0Cn7H/Ny8dazdUsW943NVUUxEDop6imDaXgTv3A85p8Hwa4LfXFUdf3x/Bd/on87pR2cEvT0RCQ9KBMHiHLxxC7gmmPCnkMwl9If3V1BRU8+943M1dYSIHLTDSgRmltzagYSdL56DVR/AOT+DzkcFvbnVpZU8N3cdk0b2ZmD31KC3JyLh43DPCApaNYpws6MEZt0L2d+AEdeGpMlfvrWU+Jgo7jjn6JC0JyLhY5/PEZjZHfv6CNAZwb44B2/eBk0NMOGPEBX80bdPV5XxbsEm7jpvABkp8UFvT0TCy/56qV8AnYGUZq/kA/xeZFv4Eqx4B866H7r0CXpzjU2Oh94sJLNTIteekhP09kQk/OzvyeLPgdecc/nNPzCz/zmYnZvZGOAPQDTwV+fcI80+fxQ4w7/YAejqnOt0MPtuk8o3wNs/gt4nwQnXh6TJVz4vpmBDOX+4fAgJsSowIyKHbn9H9iXAOjO7tYXPRhxox2YWDTwGjAUGAVeY2aDAbZxztzvnhjjnhgB/Al456MjbGudgxh3QUAsTHwvJkFBVXQO/nrWM43t1YsLxwZ22QkTC1/56q0FAHPA9M+tsZl12vYD6g9j3CcBK59xq51wd8BIwcT/bXwG8eLCBtzmLp8GymXDmfZDWNyRNPvXRajZX1PIT3S4qIkdgf0NDTwLvA32AfHwXiXdx/vX7kwkUBSwXAye2tKGZHQXkAB/s4/PJwGSA3r17H6BZD1RsgrfugqyRMOqGkDS5cUcNT364mvHH9mBEdpeQtCki4WmfZwTOuT8653KBZ5xzfZxzOQGv1r4KejkwzTnXuI9YnnLOjXDOjcjIaGNPzDoHM38AdVUw8XGICs04/W/eWUZjk+NHYwaGpD0RCV8HHMh2zv3vYe67BAicXCfLv64ll9Neh4WWvAqFb8AZ90BGaO7h/7JkBy9/Xsx3R2fTO61DSNoUkfAVzCua84H+ZpZjZnH4OvvpzTcys4H4blOdG8RYgmNnGcy8E3oOg5NuDkmTvloDBXTuEMcNZ6jWgIgcuaAlAudcA3ATMAsoBKY455aY2YNmNiFg08uBl5xzLlixBM3MO6G2wneXUHRoavy8W7CJeau3cvvZ/emYqFoDInLkgtp7OedmAjObrbu/2fIDwYwhaAqm+4aFzrwPug068PatoK6hiV++tdRXa+CENnjRXETaJT0hfDiqtvqeGeh+HIy+LWTN/mveOtaU7eTecao1ICKtRzWLD8dbP4TqbXD1axAdmuGZ7VV1/GFXrYEBbezOKRFp13RYeaiWzoTFU+HUu6D7MSFr9k8frKS8pp4fj9PDYyLSupQIDkX1Nnjzduh2DJyyr8lZW9+asp08O3ctk0b0IreHag2ISOvS0NChePvHsLMUrvw3xMSFrNlH3iokNjqKO85VrQERaX06IzhYy2fBwhfgG3dAzyEha3be6i3MWrKJG07vS9eUhJC1KyKRQ4ngYNTsgDdug4xc37WBEGlq8j081rNjAv/zjeDXNhCRyKREcDBm3QuVG+HCxyAmdBXAXv2ihC9LyvnhmIGqNSAiQaNEcCAr3/MVoh99K2QOD1mzu2sNZHVUrQERCSolgv2pKYfpt0L6ADjt7pA2/fRHa9hYXsN95w8iKkq3i4pI8Oiuof15936oWA/fewdiQ3ehdlN5DU98uIpxx3ZnpGoNiEiQ6YxgX1bPgfy/+wrN9BoZ0qZ/M0u1BkQkdJQIWlJbCdNvhrR+vknlQmjJ+h1M+7yYa0Znc1RaUkjbFpHIpKGhlrz3AGwvgu+9DbGJIWvWOcfDMwrplM841/wAAA3iSURBVBjLjao1ICIhojOC5tZ+DPOfhhO/D71HhbTp9ws38+mqLdx29tGqNSAiIaNEEKhuJ7x+E3TOgbN+EtKm6xub+MXMQvpkJHHliao1ICKho6GhQO//HLatgWtmQFxox+efn7eO1WU7+dt3RhCrWgMiEkLqcXb5ah789wkYeR1knxLSpndU1fP791cwul8aZw7sGtK2RUSUCADqq+G1G6BTLzj7gZA3/6cPVrCjup57xw1SrQERCTkNDQF88BBsXQXfng7xySFtem3ZTv45dy2XDe/FoJ6qNSAioaczgqL5MO9xGP5d6HNayJt/5K2lxEZH8QPVGhARj0R2IqivgddvgJSecM6DIW/+szVbeXvJRv73tL50TVWtARHxRmQPDX34CJQth6tegYTQDsvsqjXQQ7UGRMRjkXtGUJIPn/wBhl4N/c4KefOvLyxhUfEOfjhmAIlxqjUgIt6JzETQUAuv3QjJ3eG8h0PefHVdI796exnHZXVk4vGZIW9fRCRQZA4NffRrKC2EK6dCQseQN//X/6xmw44afj9piGoNiIjnIu+MYP0C+M/v4Pgr4ehzQ9785vIa/vLhKsYM7s6JfdJC3r6ISHORlQga6uD1GyEpA8b8wpMQfvvOcuobm7h7rGoNiEjbEFlDQx//DjZ9CZe/CImdQ958wfpypuQXce3oHLLTVWtARNqGyDkj2LjYd23g2Mtg4LiQN++c4+GZBXRMjOXmM/uHvH0RkX2JnESw9hPokA5j/8+T5mcv28wnK7dw21n96dhBtQZEpO2InKGhUd+HIVeG/MEx8NUaeHhGIX3Sk/jWqKNC3r6IyP5EzhkBeJIEAF787CtWle7kx+NyVWtARNoc9UpBtqO6nkffXc7JfdM4K1e1BkSk7QlqIjCzMWa2zMxWmtnd+9jmMjMrMLMlZvZCMOPxwmOzV7K9up57x+eq1oCItElBu0ZgZtHAY8A5QDEw38ymO+cKArbpD9wDjHbObTOzsDpkXrdlJ//4ZC2XDMticM/QP8EsInIwgnlGcAKw0jm32jlXB7wETGy2zXXAY865bQDOuc1BjCfk/u/tpURHGXeeN8DrUERE9imYiSATKApYLvavC3Q0cLSZfWJm88xsTEs7MrPJZpZnZnmlpaVBCrd1zV+7lZmLN/L90/rSTbUGRKQN8/picQzQHzgduAJ42sw6Nd/IOfeUc26Ec25ERkZGiEM8dE1NjofeLKB7agLXnZrjdTgiIvsVzERQAvQKWM7yrwtUDEx3ztU759YAy/ElhnZt+sL1LCzewV3nDaBDXOQ8qiEi7VMwE8F8oL+Z5ZhZHHA5ML3ZNq/hOxvAzNLxDRWtDmJMQVdT38iv3l7KMZmpfHOoag2ISNsXtETgnGsAbgJmAYXAFOfcEjN70Mwm+DebBWwxswJgNnCXc25LsGIKhb99vIb1O2q4b/wg1RoQkXYhqOMWzrmZwMxm6+4PeO+AO/yvdm9zRQ2Pz17JeYO7MUq1BkSknfD6YnFYefTd5dQ1NnH32FyvQxEROWhKBK2kcEM5/55fxLdPyiZHtQZEpB1RImgFzjkenlFISkIsN5/Zz+twREQOiRJBK5izrJSPV5Zx61n96dQhzutwREQOiRLBEapvbOKhGQXkpCdxlWoNiEg7pERwhF7y1xq4Z+xA4mL0zyki7Y96riNQXlPPo++tYFSfLpwzqJvX4YiIHBYlgiPw2OyVbKuq477xg1RrQETaLSWCw1S0tYq/f7yWi4dlcUymag2ISPulRHCYHtlVa+Bc1RoQkfZNieAw5K/byoxFG7j+tD5076haAyLSvikRHKKmJseDbxbSLTWeyaf28TocEZEjpkRwiN5YtJ6FRdu567yBqjUgImFBieAQ+GoNLGNwz1QuUq0BEQkTSgSH4G8fr6FkezX3js9VrQERCRtKBAeptKKWv8xZxTmDunFy33SvwxERaTVKBAfp0feWU1PfyD1jB3odiohIq1IiOAjLNlbw0mdfcfVJR9EnI9nrcEREWpUSwUF4eKav1sCtZ/X3OhQRkVanRHAAc5Zt5qPlpdyiWgMiEqaUCPajobGJh2cUkp3WgatVa0BEwpQSwX68NL+IFZsruWdcrmoNiEjYUu+2D+U19Tz67nJOzOnCuao1ICJhTIlgHx6fvYqtVXX85HzVGhCR8KZE0IKirVU88/Eavjk0U7UGRCTsKRG04FezlhEVBXedp1oDIhL+lAiayV+3jTcWrmfyqX3p0THR63BERIJOiSCAc46HZhTQNSWe61VrQEQihBJBgDcXbeCLr7Zz53kDSIpXrQERiQxKBH419Y088tZSBvVI5eJhWV6HIyISMkoEfn//ZC0l26u5b3wu0ao1ICIRRIkAKKus5bHZKzk7txsn91OtARGJLEoEwKPv+msNjFOtARGJPBGfCJZvquDFz77iqlFH0Ve1BkQkAgU1EZjZGDNbZmYrzezuFj6/xsxKzWyB//U/wYynJb+YWUhyfIxqDYhIxAraPZJmFg08BpwDFAPzzWy6c66g2ab/ds7dFKw49ufD5aXMWVbKfeNz6ZykWgMiEpmCeUZwArDSObfaOVcHvARMDGJ7h8RXa6CAo9I6cPVJqjUgIpErmIkgEygKWC72r2vuYjNbZGbTzKxXEOPZw5S8YpZvquSesQOJj4kOVbMiIm2O1xeL3wCynXPHAe8C/2xpIzObbGZ5ZpZXWlp6xI1W1NTzu3eXcUJ2F84b3P2I9yci0p4FMxGUAIFH+Fn+dbs557Y452r9i38Fhre0I+fcU865Ec65ERkZGUcc2F/mrKKsso77zs9VrQERiXjBTATzgf5mlmNmccDlwPTADcysR8DiBKAwiPEAULytir9+vIaLhmZyXFanYDcnItLmBe2uIedcg5ndBMwCooFnnHNLzOxBIM85Nx24xcwmAA3AVuCaYMWzy6/eXkaUwZ2qNSAiAgQxEQA452YCM5utuz/g/T3APcGMIdAXX21j+sL13HJmP3p2Uq0BERHw/mJxyPhqDRSSkRLP9af19TocEZE2I2ISwczFG8lft427zlWtARGRQBGTCDrER3PuoG5cPFy1BkREAkXMofEZA7pyxoCuXochItLmRMwZgYiItEyJQEQkwikRiIhEOCUCEZEIp0QgIhLhlAhERCKcEoGISIRTIhARiXDmnPM6hkNiZqXAusP89XSgrBXDaQ/0nSODvnNkOJLvfJRzrsWCLu0uERwJM8tzzo3wOo5Q0neODPrOkSFY31lDQyIiEU6JQEQkwkVaInjK6wA8oO8cGfSdI0NQvnNEXSMQEZG9RdoZgYiINKNEICIS4SImEZjZGDNbZmYrzexur+MJNjN7xsw2m9mXXscSKmbWy8xmm1mBmS0xs1u9jinYzCzBzD4zs4X+7/wzr2MKBTOLNrMvzOxNr2MJBTNba2aLzWyBmeW1+v4j4RqBmUUDy4FzgGJgPnCFc67A08CCyMxOBSqBZ51zx3gdTyiYWQ+gh3PuczNLAfKBC8P8v7MBSc65SjOLBT4GbnXOzfM4tKAyszuAEUCqc+58r+MJNjNbC4xwzgXlAbpIOSM4AVjpnFvtnKsDXgImehxTUDnnPgK2eh1HKDnnNjjnPve/rwAKgUxvowou51PpX4z1v8L66M7MsoDxwF+9jiVcREoiyASKApaLCfMOItKZWTYwFPivt5EEn3+YZAGwGXjXORfu3/n3wA+BJq8DCSEHvGNm+WY2ubV3HimJQCKImSUDLwO3OefKvY4n2Jxzjc65IUAWcIKZhe1QoJmdD2x2zuV7HUuIneKcGwaMBW70D/22mkhJBCVAr4DlLP86CTP+cfKXgeedc694HU8oOee2A7OBMV7HEkSjgQn+MfOXgDPN7F/ehhR8zrkS/8/NwKv4hrtbTaQkgvlAfzPLMbM44HJguscxSSvzXzj9G1DonPud1/GEgpllmFkn//tEfDdELPU2quBxzt3jnMtyzmXj+zv+wDl3lcdhBZWZJflvfsDMkoBzgVa9GzAiEoFzrgG4CZiF7wLiFOfcEm+jCi4zexGYCwwws2Izu9brmEJgNHA1vqPEBf7XOK+DCrIewGwzW4TvgOdd51xE3FIZQboBH5vZQuAzYIZz7u3WbCAibh8VEZF9i4gzAhER2TclAhGRCKdEICIS4ZQIREQinBKBiEiEUyIQCSEzOz1SZsyU9kOJQEQkwikRiLTAzK7yz/O/wMye9E/sVmlmj/rn/X/fzDL82w4xs3lmtsjMXjWzzv71/czsPX+tgM/NrK9/98lmNs3MlprZ8/4nokU8o0Qg0oyZ5QKTgNH+ydwagW8BSUCec24w8CHwU/+vPAv8yDl3HLA4YP3zwGPOueOBk4EN/vVDgduAQUAffE9Ei3gmxusARNqgs4DhwHz/wXoivimem4B/+7f5F/CKmXUEOjnnPvSv/ycw1T83TKZz7lUA51wNgH9/nznniv3LC4BsfAVlRDyhRCCyNwP+6Zy7Z4+VZj9ptt3hzs9SG/C+Ef0disc0NCSyt/eBS8ysK4CZdTGzo/D9vVzi3+ZK4GPn3A5gm5l9w7/+auBDf4W0YjO70L+PeDPrENJvIXKQdCQi0oxzrsDM7sNXESoKqAduBHbiK/xyH76hokn+X/kO8IS/o18NfNe//mrgSTN70L+PS0P4NUQOmmYfFTlIZlbpnEv2Og6R1qahIRGRCKczAhGRCKczAhGRCKdEICIS4ZQIREQinBKBiEiEUyIQEYlw/w+UVj6aScI1RQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n2Q1B_RT6Pyv",
        "colab_type": "text"
      },
      "source": [
        "Судя по графику в модели наблюдается небольшое переобучение, но f1-мера 0.88 на тесте оказалась на несколько процентов лучше, чем в бейзлайне на всех данных 0.83 и почти такой же как в бейзлайне на даунсэмпле 0.89."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "99gghnKB3-qd",
        "colab_type": "text"
      },
      "source": [
        "## 5. Использование токенизации на символьные нграммы (не параметром analyzer=’char’, а вручную или готовым энструментов вроде youtokentome - 2 балла"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jlvijey7h8O4",
        "colab_type": "text"
      },
      "source": [
        "С использованием кода по youtokentome Алексея Доркина на даунсэмпле"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wngfwz02gU6s",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "5805fd62-bb21-45fc-99c0-bff4f3a981a5"
      },
      "source": [
        "cd '/gdrive/My Drive/exam/'"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/gdrive/My Drive/exam\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uw0Y07MOgUzJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train, test = train_test_split(data_downsampled, test_size=0.3, random_state=0) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "imO7GzFPgUr4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "comments = train.comment_text.to_csv(index=False, header=None) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vVv3qEWpgUjb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('/gdrive/My Drive/exam/comments.txt', 'w') as file: \n",
        "  new_comments = re.sub(r'\\\"{2,}', '', comments) \n",
        "  new_comments = re.sub(r'\\n+', ' ', new_comments) \n",
        "  file.write(new_comments)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "foCzovgJgUbt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "b0218c2b-a50a-4149-ef58-7491ab9c868c"
      },
      "source": [
        "%%time\n",
        "yttm.BPE.train('/gdrive/My Drive/exam/comments.txt', '/gdrive/My Drive/exam/bpe.model', vocab_size=2000, coverage=0.9999, n_threads=-1, pad_id=0, unk_id=1) "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 1.55 s, sys: 90.9 ms, total: 1.64 s\n",
            "Wall time: 1.21 s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<youtokentome.youtokentome.BPE at 0x7f29a9c7d4e0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 128
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M646_PWHiOsN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bpe_model = yttm.BPE('/gdrive/My Drive/exam/bpe.model', n_threads=-1) # читаем модель из файла"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zeYG-Ku0iYYW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "encoded_comments = bpe_model.encode(data_downsampled.comment_text.tolist(), output_type=yttm.OutputType.ID) # на выходе получаем сразу ID подслов в предложениях, что нам и нужно на входе в сеть"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UDQq-93KgUTx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data_downsampled['encoded_comments'] = encoded_comments # записываем закодированные версии для всех предложений"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NsMxOKzAidI5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train, test = train_test_split(data_downsampled, test_size=0.3, random_state=0) # опять делим на трейн и тест (с тем же сидом, чтобы разделение было такое же)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ijKSgLqhig18",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "910cd63b-cba8-47d8-aecd-61d0ad62f699"
      },
      "source": [
        "[bpe_model.id_to_subword(num) for num in encoded_comments[0]][:10] # модель содержит словарь подслов"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['▁h', 'mm', 'mm', '...', '▁app', 'are', 'nt', 'ly', '▁i', \"▁didn't\"]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 133
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "psx5t58MifEs",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "6a9154cb-5aac-4ef1-eabc-9ebee6fb9aba"
      },
      "source": [
        "MAX_LEN = max(len(text) for text in train['encoded_comments'])\n",
        "MAX_LEN"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4982"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 134
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XMxMcAk0ikF0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "69ada9f4-5ec0-4155-b3a0-a3ee3b69ad33"
      },
      "source": [
        "MEAN_LEN = np.median([len(text) for text in train['encoded_comments']])\n",
        "MEAN_LEN"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "54.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 135
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LIBBWMyD4e92",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "8ffaeae1-5d85-40cc-ac2e-02c6a0299450"
      },
      "source": [
        "len(bpe_model.vocab())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 136
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lb7dk8I801sZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "MAX_LEN = 60\n",
        "X_train = tf.keras.preprocessing.sequence.pad_sequences(train['encoded_comments'], maxlen=MAX_LEN)\n",
        "X_valid = tf.keras.preprocessing.sequence.pad_sequences(test['encoded_comments'], maxlen=MAX_LEN)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YbYQ8ptZ2KEx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_train = train['toxic']\n",
        "y_valid = test['toxic']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yW4Ga5gN2rSh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "checkpoint = tf.keras.callbacks.ModelCheckpoint('model.weights', \n",
        "                                                monitor='val_f1', \n",
        "                                                verbose=1, \n",
        "                                                save_weights_only=True, \n",
        "                                                save_best_only=True,\n",
        "                                                mode='max', \n",
        "                                                save_freq='epoch' \n",
        "                                               )\n",
        "\n",
        "early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_f1', \n",
        "                                              min_delta=0.01, \n",
        "                                              patience=3, \n",
        "                                              verbose=1, \n",
        "                                              mode='max',\n",
        "                                              )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YZ06lpUV2yM0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "inputs = tf.keras.layers.Input(shape=(MAX_LEN,))\n",
        "embeddings = tf.keras.layers.Embedding(input_dim=len(bpe_model.vocab()), output_dim=50)(inputs, )\n",
        "convs = []\n",
        "\n",
        "conv1 = tf.keras.layers.Conv1D(kernel_size=2, filters=24, strides=1)(embeddings)\n",
        "drop1 = tf.keras.layers.Dropout(0.1)(conv1)\n",
        "conv2 = tf.keras.layers.Conv1D(kernel_size=1, filters=16, strides=1, activation='relu')(drop1)\n",
        "pool1 = tf.keras.layers.GlobalAveragePooling1D()(conv2)\n",
        "drop2 = tf.keras.layers.Dropout(0.2)(pool1)\n",
        "convs.append(drop2)\n",
        "\n",
        "conv3 = tf.keras.layers.Conv1D(kernel_size=4, filters=32, strides=1)(embeddings)\n",
        "conv4 = tf.keras.layers.Conv1D(kernel_size=6, filters=16, strides=1, activation='relu')(conv3)\n",
        "pool2 = tf.keras.layers.GlobalMaxPooling1D()(conv4)\n",
        "drop3 = tf.keras.layers.Dropout(0.3)(pool2)\n",
        "convs.append(drop3)\n",
        "\n",
        "conv5 = tf.keras.layers.Conv1D(kernel_size=8, filters=32, strides=1, kernel_regularizer='l2')(embeddings)\n",
        "conv6 = tf.keras.layers.Conv1D(kernel_size=10, filters=16, strides=1, activation='relu')(conv5)\n",
        "pool3 = tf.keras.layers.GlobalAveragePooling1D()(conv6)\n",
        "drop4 = tf.keras.layers.Dropout(0.4)(pool3)\n",
        "convs.append(drop4)\n",
        "\n",
        "conv7 = tf.keras.layers.Conv1D(kernel_size=2, filters=16, strides=2, kernel_regularizer='l2')(embeddings)\n",
        "conv8 = tf.keras.layers.Conv1D(kernel_size=2, filters=16, strides=2, activation='relu')(conv7)\n",
        "pool4 = tf.keras.layers.GlobalMaxPooling1D()(conv8)\n",
        "drop5 = tf.keras.layers.Dropout(0.5)(pool4)\n",
        "convs.append(drop5)\n",
        "\n",
        "concat = tf.keras.layers.concatenate(convs)\n",
        "\n",
        "dense = tf.keras.layers.Dense(64, activation='relu')(concat)\n",
        "\n",
        "outputs = tf.keras.layers.Dense(1, activation='sigmoid')(dense)\n",
        "\n",
        "model = tf.keras.Model(inputs=inputs, outputs=outputs)\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
        "model.compile(optimizer=optimizer,\n",
        "              loss='binary_crossentropy',\n",
        "              metrics=[f1])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uqx4OVHM4tym",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 731
        },
        "outputId": "95ac5434-89e8-4b30-f977-d7db7a335f0f"
      },
      "source": [
        "model.fit(X_train, y_train, \n",
        "          validation_data=(X_valid, y_valid),\n",
        "          batch_size=2000,\n",
        "         epochs=15,\n",
        "         callbacks=[checkpoint, early_stop])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/15\n",
            "15/15 [==============================] - ETA: 0s - loss: 1.2205 - f1: 0.5272\n",
            "Epoch 00001: val_f1 improved from -inf to 0.62807, saving model to model.weights\n",
            "15/15 [==============================] - 14s 910ms/step - loss: 1.2205 - f1: 0.5272 - val_loss: 1.0964 - val_f1: 0.6281\n",
            "Epoch 2/15\n",
            "15/15 [==============================] - ETA: 0s - loss: 0.9972 - f1: 0.6406\n",
            "Epoch 00002: val_f1 improved from 0.62807 to 0.69330, saving model to model.weights\n",
            "15/15 [==============================] - 13s 878ms/step - loss: 0.9972 - f1: 0.6406 - val_loss: 0.8687 - val_f1: 0.6933\n",
            "Epoch 3/15\n",
            "15/15 [==============================] - ETA: 0s - loss: 0.7483 - f1: 0.7649\n",
            "Epoch 00003: val_f1 improved from 0.69330 to 0.81240, saving model to model.weights\n",
            "15/15 [==============================] - 13s 879ms/step - loss: 0.7483 - f1: 0.7649 - val_loss: 0.6201 - val_f1: 0.8124\n",
            "Epoch 4/15\n",
            "15/15 [==============================] - ETA: 0s - loss: 0.5717 - f1: 0.8263\n",
            "Epoch 00004: val_f1 improved from 0.81240 to 0.84351, saving model to model.weights\n",
            "15/15 [==============================] - 13s 878ms/step - loss: 0.5717 - f1: 0.8263 - val_loss: 0.5011 - val_f1: 0.8435\n",
            "Epoch 5/15\n",
            "15/15 [==============================] - ETA: 0s - loss: 0.4626 - f1: 0.8569\n",
            "Epoch 00005: val_f1 improved from 0.84351 to 0.85933, saving model to model.weights\n",
            "15/15 [==============================] - 13s 878ms/step - loss: 0.4626 - f1: 0.8569 - val_loss: 0.4293 - val_f1: 0.8593\n",
            "Epoch 6/15\n",
            "15/15 [==============================] - ETA: 0s - loss: 0.3979 - f1: 0.8699\n",
            "Epoch 00006: val_f1 improved from 0.85933 to 0.86921, saving model to model.weights\n",
            "15/15 [==============================] - 13s 878ms/step - loss: 0.3979 - f1: 0.8699 - val_loss: 0.3880 - val_f1: 0.8692\n",
            "Epoch 7/15\n",
            "15/15 [==============================] - ETA: 0s - loss: 0.3566 - f1: 0.8780\n",
            "Epoch 00007: val_f1 improved from 0.86921 to 0.87059, saving model to model.weights\n",
            "15/15 [==============================] - 13s 884ms/step - loss: 0.3566 - f1: 0.8780 - val_loss: 0.3640 - val_f1: 0.8706\n",
            "Epoch 8/15\n",
            "15/15 [==============================] - ETA: 0s - loss: 0.3305 - f1: 0.8827\n",
            "Epoch 00008: val_f1 improved from 0.87059 to 0.87194, saving model to model.weights\n",
            "15/15 [==============================] - 13s 877ms/step - loss: 0.3305 - f1: 0.8827 - val_loss: 0.3503 - val_f1: 0.8719\n",
            "Epoch 9/15\n",
            "15/15 [==============================] - ETA: 0s - loss: 0.3090 - f1: 0.8896\n",
            "Epoch 00009: val_f1 did not improve from 0.87194\n",
            "15/15 [==============================] - 13s 873ms/step - loss: 0.3090 - f1: 0.8896 - val_loss: 0.3409 - val_f1: 0.8718\n",
            "Epoch 10/15\n",
            "15/15 [==============================] - ETA: 0s - loss: 0.2939 - f1: 0.8904\n",
            "Epoch 00010: val_f1 did not improve from 0.87194\n",
            "15/15 [==============================] - 13s 870ms/step - loss: 0.2939 - f1: 0.8904 - val_loss: 0.3375 - val_f1: 0.8701\n",
            "Epoch 00010: early stopping\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f29bc256f60>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 141
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TSoNiRBg8Xib",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        },
        "outputId": "402e0257-5e08-4789-cde9-a337be6702ad"
      },
      "source": [
        "print(model.history.history.keys())\n",
        "plt.plot(model.history.history['f1'])\n",
        "plt.plot(model.history.history['val_f1'])\n",
        "plt.title('model f1')\n",
        "plt.ylabel('f1')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'val'], loc='upper left')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dict_keys(['loss', 'f1', 'val_loss', 'val_f1'])\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9bnH8c+TjSxAEgIIJIFEBVkERSJat2LdQKt4rQvW9Xaxm7Xa7WIXa6ltbe2KpVWs9NpWRatV6ZUWF8ClqAXcQZawh7AEEhBIIMs8948ZYAgBJ2ROJsv3/XrllTPnnN+Zh9Gc7/zO8jvm7oiIiDSWlOgCRESkbVJAiIhIkxQQIiLSJAWEiIg0SQEhIiJNUkCIiEiTFBAiATGz/zWzu2Jcd7WZnXuIZRlm9g8z225mf4tvlSKHpoAQafsuB44C8tz9CjPra2YzzKzczNzMihJbnnRUCgiRtm8AsMzd6yOvQ8C/gE8lriTpDBQQ0qlFDu18y8zeNbNdZvagmR1lZv80sx1m9oKZ5Uatf4mZLTKzbWY218yGRC0baWZvRto9BqQ3eq9PmtnbkbbzzGxEDPX9ELgDuMrMdprZZ919k7v/Hpgfv09C5GAKCJHwN/HzgEHAxcA/ge8AvQj/jdwCYGaDgEeBWyPLZgL/MLM0M0sDngb+AvQA/kbUN3wzGwlMA74A5AH3AzPMrMvhCnP3HwA/AR5z967u/mCc/s0iH0kBIQL3Rr6VrwdeAd5w97fcfTfwFDAyst5VwLPu/ry71wG/ADKA04BTgVTgN+5e5+5PcOA3/JuA+939DXdvcPeHgD2RdiJtUkqiCxBpAzZFTdc08bprZLofsGbvAncPmdk6IB9oANb7gaNfromaHgDcYGZfjZqXFtmmSJukgBCJXTkwfO8LMzOgEFgPOJBvZhYVEv2BFZHpdcCP3f3HrVivSIvoEJNI7B4HLjKzc8wsFfgG4cNE84DXgHrgFjNLNbPLgNFRbR8Avmhmp1hYlpldZGbdjqQQM0sH9p6/6BJ5LRJXCgiRGLn7UuBa4F5gC+ET2he7e6271wKXATcClYTPV/w9qu0C4PPA74AqoDSy7pGqAXZGppdEXovElemBQSIi0hT1IEREpEkKCBERaZICQkREmqSAEBGRJnWY+yB69uzpRUVFiS5DRKRdWbhw4RZ379XUsg4TEEVFRSxYsCDRZYiItCtmtuZQywI9xGRmY81sqZmVmtnEJpYPMLMXIyNpzjWzgqhlN5jZ8sjPDUHWKSIiBwssIMwsGZgCjAOGAleb2dBGq/0C+LO7jwAmAT+NtO0B/AA4hfDdqD+IHnJZRESCF2QPYjRQ6u4rI3eZTgfGN1pnKDA7Mj0navkFwPPuXunuVcDzwNgAaxURkUaCPAeRT3iAsr3KCPcIor1DeHiC3wL/BXQzs7xDtM1v/AZmdhPhYZTp37//QQXU1dVRVlbG7t27j/xf0U6kp6dTUFBAampqoksRkQ4i0Sepvwn8zsxuBF4mPCpmQ6yN3X0qMBWgpKTkoDFDysrK6NatG0VFRYQH3uyY3J2tW7dSVlZGcXFxossRkQ4iyENM6wkPhbxXQWTePu5e7u6XuftI4LuRedtiaRuL3bt3k5eX16HDAcDMyMvL6xQ9JRFpPUEGxHxgoJkVRx7HOAGYEb2CmfU0s7013E74kYwAs4DzzSw3cnL6/Mi8Zuvo4bBXZ/l3ikjrCewQk7vXm9nNhHfsycA0d19kZpOABe4+AxgD/NTMnPAhpq9E2laa2Y/Y/8jGSe5eGVStIiLNVVsfYntNHQ0hp8GdUMipDzkNISfk4d97p+tD4eX714X6UCiyHge1id5mQ6P5e9uE3KlvCC/v0z2dT59y8HnYlgr0HIS7zyT8YPfoeXdETT8BPHGIttPY36Not7Zt28YjjzzCl7/85Wa1u/DCC3nkkUfIyckJqDIROZS6hhCbPtzNxu27Kd++mw3batiwfTcbtu/9vZstO/fQVp6WMLJ/TvsLCAkHxO9///uDAqK+vp6UlEN//DNnzjzkMhE5cvUNITbv2LN/h79t977p8u272bi9hoodewg12vlnpSXTNyeDvtnpDOnTnT7Z6fTsmkZyUhLJSZBkRkqykWRGcpKRbEZSkpGSFP6dHJm/b3lS9HqQvHfdw663f3rv/CQL7hCzAiJgEydOZMWKFZx44omkpqaSnp5Obm4uS5YsYdmyZVx66aWsW7eO3bt387WvfY2bbroJ2D90yM6dOxk3bhxnnHEG8+bNIz8/n2eeeYaMjIwE/8tE2p6GkFOxY8++b/rl22rYGPnGX749PL3pw90H7fwz05Lpm51O3+wMBg3stS8I+man0y8ngz7Z6XRP73yXkHeagPjhPxaxuPzDuG5zaL/u/ODiYYdd5+677+b999/n7bffZu7cuVx00UW8//77+y5HnTZtGj169KCmpoaTTz6ZT33qU+Tl5R2wjeXLl/Poo4/ywAMPcOWVV/Lkk09y7bXXxvXfItKW1TWE2FZdx7bqWqqq66jctYfybbvZ+GE4BDZs371v51/faO+fnppEv+wM+uakc9oxPemXk06f7PR98/p2z6B7Roou9GhCpwmItmL06NEH3KswefJknnrqKQDWrVvH8uXLDwqI4uJiTjzxRABGjRrF6tWrW61ekXhyd2rqGqiqrqNqVy3bquuoqq5lW3Utlbv2T1dFhUHVrlp27KlvcntdUpL2ffM/pbhHeIefnbFvXr+cdLIzUrXzP0KdJiA+6pt+a8nKyto3PXfuXF544QVee+01MjMzGTNmTJP3MnTp0mXfdHJyMjU1ej69JF4o5Hy4uy68E9+7Y4/s5Kuid/L7dvzh33vqQ4fcZrcuKeRkpZKbmUZuZhrFPbPIiUznZqWSk5lGj8h03+wMcjO18w9SpwmIROnWrRs7duxoctn27dvJzc0lMzOTJUuW8Prrr7dydSJNC4WcjR/uZl1lNWsrq1lXVcO6ymrKqqrZGvnmv6269qBj+XslJxk5GankZIZ39oU9MhlREJ4O7/BT9/3ukRWel5OZSmqynmHWliggApaXl8fpp5/O8ccfT0ZGBkcdddS+ZWPHjuW+++5jyJAhHHfccZx66qkJrFQ6m+01dayrrA7/VIWDYG1lDWWV1ZRV1VDbsP+bfpJB3+wMCnIzGNKnO7lZB+7scyM7+L07+25dUkhK0jf79s68rVzI20IlJSXe+IFBH3zwAUOGDElQRa2vs/175fBq60OUb6uJ9AAiPYHKatZVhudtr6k7YP3sjFT698ikf49MCnpk7JsuzM2kX04GaSn6dt8RmdlCdy9papl6ECLtlLuzZWctayOHftZurd4XBusqa9iwveaAQ0BpyUkU5GZQ2COTEwqz94dBbiaFPTLJzgjoMs5QCEJ10FALDXXhn8avG2ohVB/7OqG9J60jvZR95yHswOl9yw41zRG0j1rPkiA5DVK6QHIXSElr9LtL1PKo9ZLbx663fVQp0olt2bmH99ZvZ1XFLsoqd1C+dQcbqz5kU9UOGupqSaWeVKsnlQZ6ZyVR1D2F045Kpu8xKfTpmkTvzCR6ZxnZaUZSaFtkBxvZ6W6vhcpaWLZ35xy9o66Fhujpuv3tGuoazYtaN9Roucc8QHPnYUkxBEkzgie7EEZcEfcyFRAibciO6hqWL32fTSvfY3f5YrpUldK3fh0nWTlnUUOyRXUJkiM/0eqByshPcyWlQvLen7So12n75yel7t8xdem6f9ne+ckpjdqmxrBOE9s/1OvkNEhKCf+YER7rIvKZHDRN+HX0YfTo+U22OVx7P7h9qCEchPV7on7vgfraRr/3HNl61bsOv/7enlTByQoIkQ6jdhd7Ni5lw4p32b7ufahYRvddq+jXUM5Jtv+a/23JPdmVdzQ1vU8lNbc3GenpWEpa0zvOQ+1QD5rfeIcbmdblou3P3oAKBdNLU0CIBGnXFtiyjNDmpWxb+x57Ni4hfdsKcus20gUoAhrcKLM+VGYUsTXvbLrmD6HvsSeSXTiUnPRsNFyjHFJSMiQFN+yOAkKkpUIh2L4WtiyHiqV4xVL2bFyCbV1Gl9ptQPjBKxmeRrn3Y40dS3X380ntM5ieRcM5+rgR9M/LZoC+wUsbo4BoY7p27crOnTsTXYY0pX4PbC2FLcugYhlsWQpbluFbSrH6/Xe3V9GN5aF8VoROYrXlU99jIN0KhlF0zCBGFPZgXF6W7hGQdkEBIdKUut1Q9h9Y/SpseDccBlWrwfffPLYlpQ/LQv1YXDuGUs9nJfmQN5Di/gM4oTCHEQXZXNGnm+4OlnZLARGwiRMnUlhYyFe+8hUA7rzzTlJSUpgzZw5VVVXU1dVx1113MX78+ARX2snV18L6hbDqZVj9Cqz7DzTswS2JbZnFrEkq5N2UEhbs6s0K78cK70ufvFxGFISD4PLCHIb1yyYjrfFlRSLtV+cJiH9OhI3vxXebfYbDuLsPu8pVV13Frbfeui8gHn/8cWbNmsUtt9xC9+7d2bJlC6eeeiqXXHKJBh1rTQ31UP4WrH4ZVr0C696AumrAoM9wtgy9nukVA7h/dR921GRyVPcujCjM4YSCbK4ozGFEfg7ZmZ3v+QDSuQQaEGY2Fvgt4au1/+judzda3h94CMiJrDPR3WeaWRHwAbA0surr7v7FIGsNysiRI9m8eTPl5eVUVFSQm5tLnz59uO2223j55ZdJSkpi/fr1bNq0iT59+iS63I4r1AAb3w2HwepXYM1rUBsZRLH3UBh5HRSfxaLU4/n1v7fwwvxNdEtP4b/PKebq0YX0zdYDmqTzCSwgzCwZmAKcB5QB881shrsvjlrte8Dj7v4HMxtK+PnVRZFlK9z9xLgV9BHf9IN0xRVX8MQTT7Bx40auuuoqHn74YSoqKli4cCGpqakUFRU1Ocy3tEAoBJsX7z9ktObfsHt7eFnPQTDiSig+EwacAV178dbaKia/uJw5SxeRnZHK188bxA2nFQU3/IRIOxBkD2I0UOruKwHMbDowHogOCAe6R6azgfIA60mYq666is9//vNs2bKFl156iccff5zevXuTmprKnDlzWLNmTaJLbP/coWJpOAxWvRw+uVwTuZ04txiGjoeis6DoDOjed1+zhWsq+e3j/+HlZRXkZKbyrQuO4/qPDaBbJ3y8pEhjQQZEPrAu6nUZcEqjde4EnjOzrwJZwLlRy4rN7C3gQ+B77v5K4zcws5uAmwD69+8fv8rjbNiwYezYsYP8/Hz69u3LNddcw8UXX8zw4cMpKSlh8ODBiS6x/XGHypX7ewirXoFdm8PLsgth0NhwD6HoTMgpPKj5Gyu3Mnn2cv5dupW8rDQmjhvMtacOoGuXznNaTuSjJPqv4Wrgf939l2b2MeAvZnY8sAHo7+5bzWwU8LSZDXP3Ax4q7e5TgakQHu67tYtvjvfe23+CvGfPnrz22mtNrqd7IA6jas2BgbAj0uHs2geOHrM/EHKLmhw2wt15beVWfvvCct5YVUnPrl347oVDuObU/mSmJfpPQaTtCfKvYj0Q/dWtIDIv2meBsQDu/pqZpQM93X0zsCcyf6GZrQAGAQuQzmP7+v1hsPpl2LY2PD+z5/4wKD4L8o497DhC7s6rpVuY/OJy5q+uone3LtzxyaFcPbq/LksVOYwgA2I+MNDMigkHwwTg043WWQucA/yvmQ0B0oEKM+sFVLp7g5kdDQwEVgZYq7QV7rByLrz0M1gb6WWl54TPHXzs5nAg9Boc08By7s5LyyqY/OJy3ly7jb7Z6UwaP4wrSwpJT1UwiHyUwALC3evN7GZgFuFLWKe5+yIzmwQscPcZwDeAB8zsNsInrG90dzezs4BJZlYHhIAvuvuRDGCMu3eK+wva/ZMB9wbD3Lth3evQrR+ceycccw4cdTwkxX43srsze8lmJr+4nHfKtpOfk8Fdlx7PFSUFdElRMIjEqkM/cnTVqlV069aNvLy8Dh0S7s7WrVvZsWMHxcXFiS6nedxh1UvhYFj7WjgYzvw6nHR9+JkDzdqU8/ziTUyevZz3139IYY8MvjLmWC47qUCPyxQ5hE77yNGCggLKysqoqKhIdCmBS09Pp6CgINFlxK6pYLjwF0cUDKGQM2vRRibPLuWDDR8yIC+Tey4fwaUj8zUOkkgLdOiASE1NbX/fqDu6QwXDyOsgNb1Zm2oIOf98fwP3vljK0k07OLpnFr+68gQuOaEfKQoGkRbr0AEhbYh7+BLVuXfD2nktDob/e7ece2eXUrp5J8f27spvJ5zIJ0f0I1nDaIvEjQJCghXHYKhvCDHjnXJ+N7uUlVt2cdxR3Zjy6ZMYd3wfPV9BJAAKCAnGQcHQ94iDoa4hxFNvrWfKnFLWbK1mSN/u3HftSZw/VMEgEiQFhMSXe/jmtrl3hwfI69YXxt0TPvnczGCorQ/x9zfLmDK3lHWVNQzPz+aB60s4d0jvDn1VmkhboYCQ+IhjMAA8t2gjP/zHYtZvq+GEwhx+eMkwzj5OwSDSmhQQ0nJ7DyXFIRgAnnl7Pbc99jaD+3TnJ5cN56yBPRUMIgmggJAjt2pvj+HVuAQDwFNvlfGNx9+hpKgHf7rxZLI0uqpIwuivT5ovOhi69oFxP4eTbmhRMAA8ubCMbz7xDqcW5/HgjSUaYVUkwfQXKLELKBgAHl+wjv958l1OOyaPP15/skZZFWkDFBDy0Va9Eh5ddfUrcQ8GgMfmr2Xi39/jjGN78sD1JRppVaSNUEDIoa1+NdxjCCgYAB55Yy3feeo9zhrUi6nXjVI4iLQhCgg5WCsEA8BfXl/D959+n7OP68UfrlU4iLQ1Cgg50H8egJnfDAfD2J/BqBsgNSPub/Pn11ZzxzOLOGdwb35/7Ul6ToNIG6SAkP327IC5Pw0/yvOavwUSDAB/+vcqfviPxZw75CimXDNS4SDSRikgZL/X74PqrXDeDwMLhz++spK7nv2AC4Ydxb1Xn6QH+Yi0YQoICauuhHmTYfAnIX9UIG8x9eUV/GTmEsYd34fJV4/Uw3xE2rhA/0LNbKyZLTWzUjOb2MTy/mY2x8zeMrN3zezCqGW3R9otNbMLgqxTCIfDnh1w9ncD2fwf5obD4aLhfRUOIu1EYD0IM0sGpgDnAWXAfDOb4e6Lo1b7HvC4u//BzIYCM4GiyPQEYBjQD3jBzAa5e0NQ9XZqOzaFDy8NvwKOGhr3zU+ZU8o9s5Zy8Qn9+PWVJ+hpbyLtRJB/qaOBUndf6e61wHRgfKN1HOgemc4GyiPT44Hp7r7H3VcBpZHtSRBe+SWE6mDMQZ28Fpv84nLumbWU8ScqHETamyD/WvOBdVGvyyLzot0JXGtmZYR7D19tRlvM7CYzW2BmCyoqKuJVd+eybS0smAYjr4W8Y+K66d+8sIxfPb+My0bm86srT1Q4iLQzif6LvRr4X3cvAC4E/mJmMdfk7lPdvcTdS3r16hVYkR3a3J+BJcFZ347bJt2dXz23lN+8sJzLRxVwzxUn6FnRIu1QkFcxrQcKo14XROZF+ywwFsDdXzOzdKBnjG2lpbYsh3cegVO/DNkHddCOiLvzi+eWMmXOCq4sKeDuy0bosaAi7VSQPYj5wEAzKzazNMInnWc0WmctcA6AmQ0B0oGKyHoTzKyLmRUDA4H/BFhr5zTnx5CaCWfcFpfNuTs/+1c4HK4eXahwEGnnAutBuHu9md0MzAKSgWnuvsjMJgEL3H0G8A3gATO7jfAJ6xvd3YFFZvY4sBioB76iK5jibMO7sOip8KGlrJ4t3py789N/LmHqyyu55pT+/Gj88QoHkXbOwvvj9q+kpMQXLFiQ6DLaj4evhHVvwK3vQnp2izbl7tz17Ac8+Ooqrjt1AJPGD9MjQkXaCTNb6O4lTS1L9ElqSYS1b8DyWXDGrXEJh0n/t5gHX13FjacVKRxEOhANtdHZuMPsH0FWbxh9Uws35dw5YxEPvbaGz5xezPc/OUThINKBqAfR2aycG37Ow1nfgrSsI95MKOR8/5n3eei1NXz+TIWDSEekHkRn4g4vToLswvBzHo5QKOR89+n3efQ/a/nCx49m4tjBCgeRDkgB0ZkseRbK34TxUyClyxFtIhRyvvPUe0yfv44vjzmGb11wnMJBpINSQHQWoYbwfQ95A2HEhCPaREPImfjku/xtYRlf/cSxfP28QQoHkQ5MAdFZvP8kbF4Ml/8Jkpv/n70h5HzriXf4+5vr+do5A7n13IEKB5EOTgHRGTTUwZyfQJ/hMPTS5jcPOd/82zs89dZ6bjt3EF87d2AARYpIW6OA6Aze+itUrYJPPw5Jzbtwrb4hxDf+9g7PvF3ON88fxM2fUDiIdBYKiI6ubje89HMoPAUGnt+spvUNIW57/B3+8U453x57HF8ec2xARYpIW6SA6OgWPAg7yuGyqdCMcwZ1DSFunf42z763gdvHDeYLH4/vsyJEpO1TQHRke3aEnxZ39NlQfGbMzeoaQtzy6Fv88/2NfO+iIXzuzKMDLFJE2ioFREf2+n1QvRU+8f2Ym9TWh7j5kTd5bvEm7vjkUD5zRnGABYpIW6aA6KiqK2HeZBj8SSgYFXOz381eznOLN/HDS4Zxw2lFwdUnIm2exmLqqOZNDh9iOvu7MTfZVl3LtH+v5sLhfRQOIqKA6JB2bAofXhp+BRw1NOZmD766ip176rnlHF3KKiIKiI7plV9CqA7GTIy5ybbqWv4U6T0M7tM9wOJEpL1QQHQ029bCgmkw8lrIi/3SVPUeRKSxQAPCzMaa2VIzKzWzg77OmtmvzeztyM8yM9sWtawhatmMIOvsUF76GVhS+FnTMVLvQUSaEthVTGaWDEwBzgPKgPlmNsPdF+9dx91vi1r/q8DIqE3UuPuJQdXXIW1ZDm8/Aqd8CbLzY26m3oOINCXIHsRooNTdV7p7LTAdGH+Y9a8GHg2wno5vzo8hNRPO/HrMTdR7EJFDCTIg8oF1Ua/LIvMOYmYDgGJgdtTsdDNbYGavm1nzhyDtbDa8C4ueglO/BFk9Y26m3oOIHEpbuVFuAvCEuzdEzRvg7uvN7Ghgtpm95+4rohuZ2U3ATQD9+/dvvWrbotl3QXoOfOzmmJuo9yAihxNkD2I9UBj1uiAyrykTaHR4yd3XR36vBOZy4PmJvetMdfcSdy/p1atXPGpun9a+Actnwelfg4ycmJup9yAihxNkQMwHBppZsZmlEQ6Bg65GMrPBQC7wWtS8XDPrEpnuCZwOLG7cVgB3mP0jyOoNp3wh5mbqPYjIRwnsEJO715vZzcAsIBmY5u6LzGwSsMDd94bFBGC6u3tU8yHA/WYWIhxid0df/SRRVs6F1a/AuJ9DWlbMzdR7EJGPEug5CHefCcxsNO+ORq/vbKLdPGB4kLV1CO7w4iTILoRRN8bcTL0HEYmF7qRuz5Y8C+VvhofUSOkSczP1HkQkFgqI9irUEL7vIW8gjJgQczP1HkQkVm3lMldprvefhM2L4fI/QXLs/xnVexCRWKkH0R411MGcn8BRw2Fo7PcQqvcgIs2hHkR79NZfoWoVfPpxSIo949V7EJHmUA+ivanbDS/9HApGw8DzY26m3oOINJd6EO3NggdhRzlcNhXMYm6m3oOINJd6EO3Jnh3wyq/g6DFQfGbMzdR7EJEjoYBoT16/D6q3wCfu+Oh1o6j3ICJHQgHRXlRXwrzJcNxFUDAq5mbqPYjIkVJAtBfzJocPMX3iu81qpt6DiBwpBUR7sGNT+PDS8CvgqGExN1PvQURaQgHRHrzyS2ioDY+51AzqPYhISygg2rpta2HBNDjpOsg7JvZm6j2ISAspINq6l34GlgRnfbtZzdR7EJGWOqKAMLOu8S5EmrBlObz9CJz8OcjOj7mZeg8iEg9H2oPQ091aw5wfQ0oGnHFbs5qp9yAi8XDIoTbM7OuHWgSoBxG0De/CoqfgrG9B114xN1PvQUTi5XA9iJ8AuUC3Rj9dP6KdxMPsuyA9Gz52c7OaqfcgIvFyuMH63gSedveFjReY2edi2biZjQV+CyQDf3T3uxst/zVwduRlJtDb3XMiy24AvhdZdpe7PxTLe3YIa9+A5bPgnB9ARk7MzdR7EJF4OlxArAfWmNnX3P23jZaVfNSGzSwZmAKcB5QB881shrvvO3/h7rdFrf9VYGRkugfwg8j7OLAw0rYqtn9WO+YOs38EWb3hlC80q6l6DyIST4c7VDQUSAM+Y2a5ZtZj7w9QF8O2RwOl7r7S3WuB6cD4w6x/NfBoZPoC4Hl3r4yEwvPA2Bjes/1bORdWvwJnfRPSsmJupt6DiMTb4XoQ9wMvAkcDCwmfnN7LI/MPJx9YF/W6DDilqRXNbABQDMw+TNuDrvM0s5uAmwD69+//EeW0A+7w4iTILoRRNzarqXoPIhJvh+xBuPtkdx8CTHP3o929OOrno8KhuSYAT7h7Q3MauftUdy9x95JevWK/0qfNWjoTyt+Ej/8PpHSJuZl6DyIShI+8Gsndv3SE214PFEa9LojMa8oE9h9eam7bjiHUEL5yKe9YOOHqZjVV70FEghDk5arzgYFmVmxmaYRDYEbjlcxsMOHLaV+Lmj0LOD9y7iMXOD8yr2NyhxfuhM2L4ezvQHLsT4JV70FEghLYM6ndvd7Mbia8Y08mfKhqkZlNAha4+96wmABMd3ePaltpZj8iHDIAk9y9MqhaE+7le8LPezj5czDssmY1Ve9BRIISWEAAuPtMYGajeXc0en3nIdpOA6YFVlxbMe934SE1Tvg0jLsHzD66TYR6DyISJN0RnUgL/gTPfReGjodL7oWk5v3nUO9BRIKkgEiUdx6D/7sNBp4Pl/2xWecdQL0HEQmeAiIRFs+Ap78ERWfAlX+GlLRmb0K9BxEJmgKitS1/AZ74DOSPgqunQ2pGszeh3oOItAYFRGta/So8dg30HgzX/A26HNmo6eo9iEhrUEC0lrKF8MhVkDMArnu6WaO0RlPvQURaiwKiNWx8D/76X5DVE65/Jvz7CKn3ICKtRQERtIpl8OdLIa0rXD8Duvc94k2p9yAirUkBEaSq1fDn8eGb365/BnIHtGhz6j2ISGsK9E7qTu3DcnjoEqirhhufhZ4t26mr9yAirU0BEYRdW8I9h+pKuOEZ6HN8izep3s0wYQgAABANSURBVIOItDYFRLzVVMFfLoVt6+DaJ8P3O7SQeg8ikggKiHjaswMevgIqlsLVj0LR6XHZrHoPIpIICoh4qauBR6+G9W+Gh8849ty4bFa9BxFJFAVEPNTXwmPXhe+UvmwqDPlk3Dat3oOIJIoCoqUa6uHJz0Lp83Dxb2HElXHbtHoPIpJIug+iJUIheOYr8MEMuOCnMOrGuG5evQcRSSQFxJFyh5nfhHenw9nfg499Oa6bV+9BRBIt0IAws7FmttTMSs1s4iHWudLMFpvZIjN7JGp+g5m9HfmZ0VTbhHGH578PCx6E02+Fs74Z97dQ70FEEi2wcxBmlgxMAc4DyoD5ZjbD3RdHrTMQuB043d2rzKx31CZq3P3EoOprkZd+DvPuhZM/D+fe2aznSMdCvQcRaQuC7EGMBkrdfaW71wLTgfGN1vk8MMXdqwDcfXOA9cTHvHth7k/gxGtg3M/jHg6g3oOItA1BBkQ+sC7qdVlkXrRBwCAz+7eZvW5mY6OWpZvZgsj8S5t6AzO7KbLOgoqKivhW35T5D8Jz34Ohl8LFkyEp/h+feg8i0lYk+jLXFGAgMAYoAF42s+Huvg0Y4O7rzexoYLaZvefuK6Ibu/tUYCpASUmJB1rpO9Ph2W/AwAvgsgcgOZiPTr0HEWkrguxBrAcKo14XROZFKwNmuHudu68ClhEODNx9feT3SmAuMDLAWg9v8Qx4+ktQfCZc+RCkpAXyNuo9iEhbEmRAzAcGmlmxmaUBE4DGVyM9Tbj3gJn1JHzIaaWZ5ZpZl6j5pwOLSYTlL8ATn4H8EpjwKKRmBPZW6j2ISFsS2CEmd683s5uBWUAyMM3dF5nZJGCBu8+ILDvfzBYDDcC33H2rmZ0G3G9mIcIhdnf01U+tZvWr8Ng10HsIXPM36NI1sLdS70FE2ppAz0G4+0xgZqN5d0RNO/D1yE/0OvOA4UHW9pHKFsAjV0FuEVz3NGTkBPp2U+aUsqtWvQcRaTt0J3VTNr4Hf70MsnqFwyErL9C3W1dZzUPz1vCpkwrUexCRNkMB0VjFMvjzpZDWNfwc6e59A3/LXz63FDP4xvmDAn8vEZFYKSCiVa0OPyrUkuD6GZA7IPC3fH/9dp5+u5zPnlFM3+zgToCLiDRXou+DaDs+LIeHLoG6avjvmdDz2MDf0t35ycwPyM1M5Ytjjgn8/UREmkM9CICdFeGeQ3UlXPd3OGpYq7zt3GUVzFuxlVvOGUj39NRWeU8RkVipB1FTBX/5L9i2LhwO+aNa5W0bQs7dM5cwIC+Ta04J/lCWiEhzqQfRUBceNmPCwzDgtFZ72yffLGPpph18+4LBpKXoP4OItD3qQXTtDZ+bHcjAe4dSU9vAL59bygmFOVw4vE+rva+ISHPoqyu0ajgATPv3KjZ9uIfvXjgEC2C4cBGReFBAtLKtO/fwh7krOG/oUYwu7pHockREDkkB0crunV1KTV0D/zN2cKJLERE5LAVEK1q1ZRd/fX0NV51cyLG9gxv4T0QkHhQQreieWUtIS0ni1nM1IJ+ItH0KiFby5toqZr63kZvOOpre3dITXY6IyEdSQLQCd+enMz+gZ9cufP7MoxNdjohITBQQreC5xZuYv7qK284bSFYX3XoiIu2DAiJgdQ0hfvbPJRzTK4urSgo/uoGISBuhgAjYY/PXsXLLLiaOG0JKsj5uEWk/At1jmdlYM1tqZqVmNvEQ61xpZovNbJGZPRI1/wYzWx75uSHIOoOyc089v3lhGaOLenDukN6JLkdEpFkCOyBuZsnAFOA8oAyYb2Yz3H1x1DoDgduB0929ysx6R+b3AH4AlAAOLIy0rQqq3iBMfXklW3bW8sD1gzWkhoi0O0H2IEYDpe6+0t1rgenA+EbrfB6YsnfH7+6bI/MvAJ5398rIsueBsQHWGnebP9zNAy+v5KLhfRnZPzfR5YiINFuQAZEPrIt6XRaZF20QMMjM/m1mr5vZ2Ga0xcxuMrMFZragoqIijqW33K9fWE59KMS3xx6X6FJERI5Ios+apgADgTHA1cADZpYTa2N3n+ruJe5e0qtXr4BKbL7SzTt4bP5arjllAAPyshJdjojIEQkyINYD0dd1FkTmRSsDZrh7nbuvApYRDoxY2rZZd/9zCVlpKXz1E8E/11pEJChBBsR8YKCZFZtZGjABmNFonacJ9x4ws56EDzmtBGYB55tZrpnlAudH5rV5r6/cygsfbOaLY44hr2uXRJcjInLEAruKyd3rzexmwjv2ZGCauy8ys0nAAnefwf4gWAw0AN9y960AZvYjwiEDMMndK4OqNV72DqnRNzudz55RnOhyRERaJNBxH9x9JjCz0bw7oqYd+Hrkp3HbacC0IOuLt2ff28A7Zdu55/IRpKcmJ7ocEZEWSfRJ6g6jtj7Ez/+1lMF9unHZSQWJLkdEpMUUEHHy19fXsLaymonjBpOcpJviRKT9U0DEwfaaOu6dvZwzju3Jxwe1ncttRURaQgERB/e9tIKq6jomjtOQGiLScSggWqh8Ww3TXl3Ff43M5/j87ESXIyISNwqIFvrlc8tw4BvnD0p0KSIicaWAaIHF5R/y97fK+O/TiijIzUx0OSIicaWAaIG7/7WE7umpfHmMhtQQkY5HAXGEXllewcvLKvjqJ44lOzM10eWIiMSdAuIIhELOT2cuoSA3g+s+NiDR5YiIBEIBcQSefns9izd8yLcuOI4uKRpSQ0Q6JgVEM+2ua+AXs5YyPD+bi0f0S3Q5IiKBUUA000PzVlO+fTe3XziYJA2pISIdmAKiGap21fK7OaWcfVwvTjumZ6LLEREJlAKiGX43p5Rde+qZOG5IoksREQmcAiJG6yqr+fNrq7liVCHH9emW6HJERAKngIjRPbOWkpxk3HaehtQQkc5BARGDd8u2MeOdcj53xtH0yU5PdDkiIq0i0IAws7FmttTMSs1sYhPLbzSzCjN7O/LzuahlDVHzZwRZ5+G4Oz9+9gPystL4wsePTlQZIiKtLrBnUptZMjAFOA8oA+ab2Qx3X9xo1cfc/eYmNlHj7icGVV+s5izdzBurKpk0fhjd0jWkhoh0HkH2IEYDpe6+0t1rgenA+ADfL+7qG0L8dOYSintmcfXo/okuR0SkVQUZEPnAuqjXZZF5jX3KzN41syfMrDBqfrqZLTCz183s0gDrPKQn3yxj+eadfPuC40hN1ukaEelcEr3X+wdQ5O4jgOeBh6KWDXD3EuDTwG/M7JjGjc3spkiILKioqIhrYdW19fzyuWWc1D+Hscf3ieu2RUTagyADYj0Q3SMoiMzbx923uvueyMs/AqOilq2P/F4JzAVGNn4Dd5/q7iXuXtKrV6+4Fv/gK6vYvGMP37lwiJ4zLSKdUpABMR8YaGbFZpYGTAAOuBrJzPpGvbwE+CAyP9fMukSmewKnA41Pbgdmy8493PfSCi4YdhQlRT1a621FRNqUwK5icvd6M7sZmAUkA9PcfZGZTQIWuPsM4BYzuwSoByqBGyPNhwD3m1mIcIjd3cTVT4GZ/OJydteH+PbYwa31liIibU5gAQHg7jOBmY3m3RE1fTtwexPt5gHDg6ztUFZW7OSRN9by6dH9OaZX10SUICLSJiT6JHWb8/N/LaVLShK3nDMw0aWIiCSUAiLKwjWV/GvRRr7w8WPo1a1LossREUkoBUSEu/OTmUvo3a0LnzuzONHliIgknAIiYtaijSxcU8XXzxtEZlqgp2ZERNoFBQRQ1xDiZ/9aysDeXbl8VEGiyxERaRMUEMD0/6xl1ZZdTBw3mBQNqSEiAigg2Lmnnt+8sJxTinvwicG9E12OiEib0ekPtlfvqaekKJcvjzlWQ2qIiETp9AHRu3s6919XkugyRETanE5/iElERJqmgBARkSYpIEREpEkKCBERaZICQkREmqSAEBGRJikgRESkSQoIERFpkrl7omuICzOrANa0YBM9gS1xKqe902dxIH0eB9LnsV9H+CwGuHuvphZ0mIBoKTNb4O66pRp9Fo3p8ziQPo/9OvpnoUNMIiLSJAWEiIg0SQGx39REF9CG6LM4kD6PA+nz2K9DfxY6ByEiIk1SD0JERJqkgBARkSZ1+oAws7FmttTMSs1sYqLrSSQzKzSzOWa22MwWmdnXEl1ToplZspm9ZWb/l+haEs3McszsCTNbYmYfmNnHEl1TIpnZbZG/k/fN7FEzS090TfHWqQPCzJKBKcA4YChwtZkNTWxVCVUPfMPdhwKnAl/p5J8HwNeADxJdRBvxW+Bf7j4YOIFO/LmYWT5wC1Di7scDycCExFYVf506IIDRQKm7r3T3WmA6MD7BNSWMu29w9zcj0zsI7wDyE1tV4phZAXAR8MdE15JoZpYNnAU8CODute6+LbFVJVwKkGFmKUAmUJ7geuKuswdEPrAu6nUZnXiHGM3MioCRwBuJrSShfgN8GwglupA2oBioAP4UOeT2RzPLSnRRieLu64FfAGuBDcB2d38usVXFX2cPCGmCmXUFngRudfcPE11PIpjZJ4HN7r4w0bW0ESnAScAf3H0ksAvotOfszCyX8NGGYqAfkGVm1ya2qvjr7AGxHiiMel0QmddpmVkq4XB42N3/nuh6Euh04BIzW0340OMnzOyviS0pocqAMnff26N8gnBgdFbnAqvcvcLd64C/A6cluKa46+wBMR8YaGbFZpZG+CTTjATXlDBmZoSPMX/g7r9KdD2J5O63u3uBuxcR/v9itrt3uG+IsXL3jcA6MzsuMuscYHECS0q0tcCpZpYZ+bs5hw540j4l0QUkkrvXm9nNwCzCVyFMc/dFCS4rkU4HrgPeM7O3I/O+4+4zE1iTtB1fBR6OfJlaCfx3gutJGHd/w8yeAN4kfPXfW3TAYTc01IaIiDSpsx9iEhGRQ1BAiIhIkxQQIiLSJAWEiIg0SQEhIiJNUkCItAFmNkYjxkpbo4AQEZEmKSBEmsHMrjWz/5jZ22Z2f+R5ETvN7NeRZwO8aGa9IuueaGavm9m7ZvZUZPwezOxYM3vBzN4xszfN7JjI5rtGPW/h4cgduiIJo4AQiZGZDQGuAk539xOBBuAaIAtY4O7DgJeAH0Sa/Bn4H3cfAbwXNf9hYIq7n0B4/J4NkfkjgVsJP5vkaMJ3toskTKceakOkmc4BRgHzI1/uM4DNhIcDfyyyzl+Bv0een5Dj7i9F5j8E/M3MugH57v4UgLvvBohs7z/uXhZ5/TZQBLwa/D9LpGkKCJHYGfCQu99+wEyz7zda70jHr9kTNd2A/j4lwXSISSR2LwKXm1lvADPrYWYDCP8dXR5Z59PAq+6+HagyszMj868DXoo8qa/MzC6NbKOLmWW26r9CJEb6hiISI3dfbGbfA54zsySgDvgK4YfnjI4s20z4PAXADcB9kQCIHv30OuB+M5sU2cYVrfjPEImZRnMVaSEz2+nuXRNdh0i86RCTiIg0ST0IERFpknoQIiLSJAWEiIg0SQEhIiJNUkCIiEiTFBAiItKk/weP0glv07fo2gAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lP0OVpmbXm3Z",
        "colab_type": "text"
      },
      "source": [
        "По сравнению с бейзлайном на даунсэмпле (0.89) метрика f1 в сверточной нейронной модели на символах получилась немного меньше (0.87)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ePEpYjd1MYZQ",
        "colab_type": "text"
      },
      "source": [
        "## 6. Поиск аутлаеров в данных с помощью кластеризации (нужно найти хотя бы 20 странных не одинаковых текста)  - 1.5 балла"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fz-LgNuHMWec",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sample = data_downsampled.sample(frac=0.01)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eXfcIfguNpP3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cv = CountVectorizer()\n",
        "svd = TruncatedSVD(50)\n",
        "X = cv.fit_transform(sample['comment_text'])\n",
        "X_svd = svd.fit_transform(X)\n",
        "\n",
        "y = sample['toxic']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q_kVCIbOPxx9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "be47d1a6-dd72-47de-f568-8cf78045ee4e"
      },
      "source": [
        "cluster = DBSCAN(min_samples=5, eps=0.5) \n",
        "cluster.fit(X_svd)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DBSCAN(algorithm='auto', eps=0.5, leaf_size=30, metric='euclidean',\n",
              "       metric_params=None, min_samples=5, n_jobs=None, p=None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 183
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B1uQgIPUQOdk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "labels = cluster.labels_"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dg7dysL7RPOI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sample['cluster'] = cluster.labels_"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eQVpGiB1ft48",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "5b34eaba-cf71-4677-949d-46257b695b03"
      },
      "source": [
        "set(cluster.labels_)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{-1, 0, 1}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 186
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1b3LRVlURSO5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "4b31bd47-bda4-4094-fcf6-6a40fda39858"
      },
      "source": [
        "len(set(cluster.labels_))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 187
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n4mjsxIESvFk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 437
        },
        "outputId": "c6a258de-5287-4ec8-86f6-9d9d26416e6c"
      },
      "source": [
        "sample[sample.cluster==0]\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "      <th>encoded_comments</th>\n",
              "      <th>cluster</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>197793</th>\n",
              "      <td>9866e05088c29349</td>\n",
              "      <td>im a fucking spz</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[813, 520, 1016, 1113, 67]</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>185357</th>\n",
              "      <td>66a2fb0d8f33392f</td>\n",
              "      <td>chocolate bar because jessca sucks</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[774, 1166, 664, 649, 533, 557, 903, 693, 924,...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>148183</th>\n",
              "      <td>482629b853c64d75</td>\n",
              "      <td>Coward. \\n\\nStop deleting these messages.</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[598, 588, 756, 24, 1116, 639, 850, 544, 1229,...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>127732</th>\n",
              "      <td>ab32d6a49fb44e30</td>\n",
              "      <td>Bloody bots get more annoying every day.... He...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[580, 14, 902, 18, 533, 558, 11, 872, 885, 548...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>208651</th>\n",
              "      <td>c37a28261ff09be4</td>\n",
              "      <td>Suck a fuckin dick bitch</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[582, 597, 520, 667, 522, 1222, 1198]</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>208758</th>\n",
              "      <td>c3f42a0d7c1762c7</td>\n",
              "      <td>Dude like era sucks</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[609, 830, 5, 732, 4, 527, 7, 1310]</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>221430</th>\n",
              "      <td>f74d3df5c6b4fbdd</td>\n",
              "      <td>WikiI   I   suck     ProjectBannerShell</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1960, 27, 540, 960, 615, 614, 1011, 50, 1311,...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>146825</th>\n",
              "      <td>321e5e8929a3f0dc</td>\n",
              "      <td>chicken shit cock sucking pussy bastard!</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[774, 792, 547, 936, 1465, 960, 544, 542, 916,...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10052</th>\n",
              "      <td>1a9983326bf9364f</td>\n",
              "      <td>go suck a dick \\n\\nplease 71.178.201.4</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[786, 960, 520, 1222, 1146, 1689, 56, 904, 75,...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>148828</th>\n",
              "      <td>52b800e978482ba2</td>\n",
              "      <td>Yeah. go suck dick u corrupt retarded fat pig ...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>[578, 5, 1412, 24, 786, 960, 1222, 622, 535, 5...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>173471</th>\n",
              "      <td>375db4e937834738</td>\n",
              "      <td>نحن شعب جدن مظلومين جغرافين عيراق وقع تحت عدد ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[4, 94, 163, 94, 4, 134, 140, 110, 4, 169, 104...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>94589</th>\n",
              "      <td>fce54e60831fd966</td>\n",
              "      <td>Agreed.  18:18, 29 Apr 2005 (UTC)</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[568, 20, 524, 959, 764, 74, 54, 56, 74, 29, 7...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>177357</th>\n",
              "      <td>4688ece61e2be6af</td>\n",
              "      <td>|class = Start \\n |photo = yes \\n |map = na \\n...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1035, 927, 996, 4, 53, 1116, 863, 1035, 1094,...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                      id  ... cluster\n",
              "197793  9866e05088c29349  ...       0\n",
              "185357  66a2fb0d8f33392f  ...       0\n",
              "148183  482629b853c64d75  ...       0\n",
              "127732  ab32d6a49fb44e30  ...       0\n",
              "208651  c37a28261ff09be4  ...       0\n",
              "208758  c3f42a0d7c1762c7  ...       0\n",
              "221430  f74d3df5c6b4fbdd  ...       0\n",
              "146825  321e5e8929a3f0dc  ...       0\n",
              "10052   1a9983326bf9364f  ...       0\n",
              "148828  52b800e978482ba2  ...       0\n",
              "173471  375db4e937834738  ...       0\n",
              "94589   fce54e60831fd966  ...       0\n",
              "177357  4688ece61e2be6af  ...       0\n",
              "\n",
              "[13 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 191
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "80Otz7jPX3dv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 406
        },
        "outputId": "2727b615-e2f1-46c2-e780-fcc44892a1a2"
      },
      "source": [
        "sample[sample.cluster==-1]\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "      <th>encoded_comments</th>\n",
              "      <th>cluster</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>125615</th>\n",
              "      <td>9ffb423fd34659d4</td>\n",
              "      <td>Cody Porter dies while foolishly copying Kimim...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[598, 1582, 615, 536, 735, 550, 831, 1666, 537...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>103946</th>\n",
              "      <td>2c268795b3b37b74</td>\n",
              "      <td>\"\\nYou dopy little fuck, \"\"wee curry monster\"\"...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[595, 720, 550, 639, 18, 1217, 667, 29, 595, 4...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>79325</th>\n",
              "      <td>d44cf9c28c392d29</td>\n",
              "      <td>Shut you dub drunk bastard. Have a great at yo...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[1576, 594, 546, 550, 953, 550, 12, 1911, 1971...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>189526</th>\n",
              "      <td>77754b5bd4d46ab5</td>\n",
              "      <td>Idiots need to be confronted in the language t...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[540, 15, 1326, 11, 980, 551, 573, 1578, 12, 1...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>126015</th>\n",
              "      <td>a1fe5fb23532f41f</td>\n",
              "      <td>\"\\nWikipedia has plenty of built-in mechanisms...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[595, 769, 796, 542, 541, 688, 18, 556, 533, 1...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26144</th>\n",
              "      <td>452cc619998777ce</td>\n",
              "      <td>This article does not meet wikipedia standard ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[989, 660, 922, 599, 652, 617, 1207, 1797, 756...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>52623</th>\n",
              "      <td>8cbab258d760c6b5</td>\n",
              "      <td>\"{| class=\"\"messagebox standard-talk\"\" style=\"...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[595, 91, 79, 1230, 996, 53, 43, 43, 19, 924, ...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>166601</th>\n",
              "      <td>1c4f835d65f07171</td>\n",
              "      <td>==What the fuck does this mean?== \\n\\n incompr...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[768, 45, 1712, 529, 667, 922, 632, 1355, 57, ...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>120667</th>\n",
              "      <td>8577f232edddfbc4</td>\n",
              "      <td>Did he? \\n\\nIs there any evidence that he DIDN...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[609, 589, 677, 57, 1409, 809, 748, 875, 1893,...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>158155</th>\n",
              "      <td>e980dad0b074bde0</td>\n",
              "      <td>fuck u\\nrape my penis plz!</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[667, 622, 670, 7, 745, 678, 1188, 1162, 67, 28]</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>409 rows × 10 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                      id  ... cluster\n",
              "125615  9ffb423fd34659d4  ...      -1\n",
              "103946  2c268795b3b37b74  ...      -1\n",
              "79325   d44cf9c28c392d29  ...      -1\n",
              "189526  77754b5bd4d46ab5  ...      -1\n",
              "126015  a1fe5fb23532f41f  ...      -1\n",
              "...                  ...  ...     ...\n",
              "26144   452cc619998777ce  ...      -1\n",
              "52623   8cbab258d760c6b5  ...      -1\n",
              "166601  1c4f835d65f07171  ...      -1\n",
              "120667  8577f232edddfbc4  ...      -1\n",
              "158155  e980dad0b074bde0  ...      -1\n",
              "\n",
              "[409 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 225
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jF03USc1jPTA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 227
        },
        "outputId": "72f8fa40-0acf-4cd3-c403-743961c5f8fa"
      },
      "source": [
        "sample[sample.cluster==1]\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "      <th>encoded_comments</th>\n",
              "      <th>cluster</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>8243</th>\n",
              "      <td>15eb1135c7e08836</td>\n",
              "      <td>Check the diff, swedefag</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[598, 605, 569, 529, 1289, 22, 29, 526, 23, 53...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26355</th>\n",
              "      <td>45c8a5c6592eb25c</td>\n",
              "      <td>through the fourth wall</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1866, 529, 537, 680, 644, 528, 753]</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>108379</th>\n",
              "      <td>435c0ab4787f122c</td>\n",
              "      <td>I removed the contentious section.</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[540, 1877, 529, 1470, 969, 1237, 24]</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>188860</th>\n",
              "      <td>74b1faee25f10504</td>\n",
              "      <td>the band called lockjaw SUCKS ASSSSSSSSSSSSSSS...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[529, 533, 669, 1919, 572, 1010, 61, 1703, 100...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>179377</th>\n",
              "      <td>4ebb9ac7aa2e5169</td>\n",
              "      <td>fuck the world cuz i fucked ur mother</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[667, 529, 1601, 535, 16, 67, 789, 667, 534, 6...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>174897</th>\n",
              "      <td>3cf2789290384003</td>\n",
              "      <td>STFU SHUT THE FCK UP</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1540, 44, 36, 1039, 946, 858, 587, 591, 1722]</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                      id  ... cluster\n",
              "8243    15eb1135c7e08836  ...       1\n",
              "26355   45c8a5c6592eb25c  ...       1\n",
              "108379  435c0ab4787f122c  ...       1\n",
              "188860  74b1faee25f10504  ...       1\n",
              "179377  4ebb9ac7aa2e5169  ...       1\n",
              "174897  3cf2789290384003  ...       1\n",
              "\n",
              "[6 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 226
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jz0RuCh2jRUx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}