keyword_extraction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "keyword_extraction.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "UbmTiYyeUTtf",
        "colab_type": "code",
        "outputId": "d568c1b6-c817-47ea-838d-7c42b211f482",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        }
      },
      "source": [
        "!pip install pymorphy2"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pymorphy2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a3/33/fff9675c68b5f6c63ec8c6e6ff57827dda28a1fa5b2c2d727dffff92dd47/pymorphy2-0.8-py2.py3-none-any.whl (46kB)\n",
            "\r\u001b[K     |███████                         | 10kB 20.9MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 20kB 1.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 30kB 1.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 40kB 1.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 51kB 1.7MB/s \n",
            "\u001b[?25hCollecting dawg-python>=0.7\n",
            "  Downloading https://files.pythonhosted.org/packages/6a/84/ff1ce2071d4c650ec85745766c0047ccc3b5036f1d03559fd46bb38b5eeb/DAWG_Python-0.7.2-py2.py3-none-any.whl\n",
            "Collecting pymorphy2-dicts<3.0,>=2.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/02/51/2465fd4f72328ab50877b54777764d928da8cb15b74e2680fc1bd8cb3173/pymorphy2_dicts-2.4.393442.3710985-py2.py3-none-any.whl (7.1MB)\n",
            "\u001b[K     |████████████████████████████████| 7.1MB 3.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: docopt>=0.6 in /usr/local/lib/python3.6/dist-packages (from pymorphy2) (0.6.2)\n",
            "Installing collected packages: dawg-python, pymorphy2-dicts, pymorphy2\n",
            "Successfully installed dawg-python-0.7.2 pymorphy2-0.8 pymorphy2-dicts-2.4.393442.3710985\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cWQNPX53VDi2",
        "colab_type": "code",
        "outputId": "4737b76a-6ff6-4519-f4cc-5b58f3dd6e36",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wDyenLYeSdUz",
        "colab_type": "code",
        "outputId": "8f67f2ae-cc81-480e-976d-f7bc7cb22e41",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import json, os\n",
        "import pandas as pd\n",
        "import tqdm\n",
        "import re\n",
        "from scipy.sparse import csr_matrix\n",
        "from nltk.corpus import stopwords\n",
        "from string import punctuation\n",
        "from nltk.stem.snowball import SnowballStemmer\n",
        "from string import digits\n",
        "punct = punctuation+'«»—…“”*№–'+digits\n",
        "from itertools import combinations\n",
        "import numpy as np\n",
        "from pymorphy2 import MorphAnalyzer\n",
        "from nltk.tokenize import word_tokenize\n",
        "from collections import Counter\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from pymystem3 import Mystem\n",
        "mystem_analyzer = Mystem()\n",
        "morph = MorphAnalyzer()\n",
        "stops = set(stopwords.words('russian'))\n",
        "stemmer = SnowballStemmer(\"russian\") "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Installing mystem to /root/.local/bin/mystem from http://download.cdn.yandex.net/mystem/mystem-3.1-linux-64bit.tar.gz\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2xrnOhsGVita",
        "colab_type": "code",
        "outputId": "c9ba601d-47ec-4369-a88f-6756b2aa8404",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "from google.colab import drive \n",
        "drive.mount('/gdrive')\n",
        "%cd /gdrive"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /gdrive\n",
            "/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ePvNGn4MWa3y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pd.set_option('display.max_colwidth', 1000)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VaMNUTfYWcAe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "PATH_TO_DATA = '/gdrive/My Drive/keyword_extraction'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x2ublnNqXP4z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "files = [os.path.join(PATH_TO_DATA, file) for file in os.listdir(PATH_TO_DATA)]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kkLyXBsPXXdr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data = pd.concat([pd.read_json(file, lines=True, encoding='utf-8') for file in files], axis=0, ignore_index=True, sort=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fDWRunxeX-62",
        "colab_type": "code",
        "outputId": "94a9d0a0-ca99-4407-9e4a-084ec3353d4d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "data.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(17266, 6)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8f-KpwrSYF1W",
        "colab_type": "code",
        "outputId": "2c1eb0ed-523d-417d-9cfd-37cb06f94998",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 661
        }
      },
      "source": [
        "data.head(1)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>keywords</th>\n",
              "      <th>title</th>\n",
              "      <th>url</th>\n",
              "      <th>abstract</th>\n",
              "      <th>content</th>\n",
              "      <th>summary</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[ЭТНОМИГРАЦИОННЫЕ ПРОЦЕССЫ, ДИАСПОРЫ, НОЦ МИОН ПРИ ИГУ, МЕЖРЕГИОНАЛЬНЫЙ, МЕЖДИСИЦИПЛИНАРНЫЕ ИССЛЕДОВАНИЯ]</td>\n",
              "      <td>От исследовательского центра к исследовательской сети: этномиграционные и диаспоральные исследования в НОЦ МИОН при ИГУ</td>\n",
              "      <td>https://cyberleninka.ru/article/n/ot-issledovatelskogo-tsentra-k-issledovatelskoy-seti-etnomigratsionnye-i-diasporalnye-issledovaniya-v-nots-mion-pri-igu</td>\n",
              "      <td>Данная статья посвящена развитию этномиграционных и диаспоральных исследований в научно-образовательном центре МИОН при ИГУ. Авторы описывают основные этапы формирования  междисциплинарного коллектива за последние 20 лет, основные результаты исследовательской работы  и ее перспективы.The article examines the development of diaspora, migration and ethnic studies in the Irkutsk CASE functioning under the ISU. The authors consider main periods of formation of interregional and interdisciplinary staff for recent 20 years, main results of the research work by the Irkutsk CASE and its prospects</td>\n",
              "      <td>ЭТНОПОЛИТИЧЕСКИЕ,\\nМИГРАЦИОННЫЕ И ДИАСПОРАЛЬНЫЕ ПРОЦЕССЫ\\nВ ПЕРЕСЕЛЕНЧЕСКОМ ОБЩЕСТВЕ /\\nETHNOPOLITICAL, MIGRATORY, AND DIASPORA'S\\nPROCESSES IN THE RESETTLEMENT SOCIETY\\nСерия «Политология. Религиоведение»\\n2012. № 2 (9), ч. 2. С. 5–12\\nОнлайн-доступ к журналу:\\nhttp://isu.ru/izvestia\\n\\nИЗВЕСТИЯ\\nИркутского\\nгосударственного\\nуниверситета\\n\\nУДК 314.7\\n\\nОт исследовательского центра\\nк исследовательской сети: этномиграционные\\nи диаспоральные исследования в НОЦ МИОН при ИГУ*\\nК. В. Григоричев, В. И. Дятлов\\nНаучно-образовательный центр Межрегиональный институт\\nобщественных наук при Иркутском государственном университете,\\nИркутский государственный университет, г. Иркутск\\nДанная статья посвящена развитию этномиграционных и диаспоральных исследований в научно-образовательном центре МИОН при ИГУ. Авторы описывают основные этапы формирования межрегионального междисциплинарного коллектива за\\nпоследние 20 лет, основные результаты исследовательской работы НОЦ МИОН при\\nИГУ и ее перспе...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                                                                    keywords  ... summary\n",
              "0  [ЭТНОМИГРАЦИОННЫЕ ПРОЦЕССЫ, ДИАСПОРЫ, НОЦ МИОН ПРИ ИГУ, МЕЖРЕГИОНАЛЬНЫЙ, МЕЖДИСИЦИПЛИНАРНЫЕ ИССЛЕДОВАНИЯ]  ...     NaN\n",
              "\n",
              "[1 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BdyOCyPJbk11",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def evaluate(true_kws, predicted_kws):\n",
        "    assert len(true_kws) == len(predicted_kws)\n",
        "    \n",
        "    precisions = []\n",
        "    recalls = []\n",
        "    f1s = []\n",
        "    jaccards = []\n",
        "    \n",
        "    for i in range(len(true_kws)):\n",
        "        true_kw = set(true_kws[i])\n",
        "        predicted_kw = set(predicted_kws[i])\n",
        "        \n",
        "        tp = len(true_kw & predicted_kw)\n",
        "        union = len(true_kw | predicted_kw)\n",
        "        fp = len(predicted_kw - true_kw)\n",
        "        fn = len(true_kw - predicted_kw)\n",
        "        \n",
        "        if (tp+fp) == 0:\n",
        "            prec = 0\n",
        "        else:\n",
        "            prec = tp / (tp + fp)\n",
        "        \n",
        "        if (tp+fn) == 0:\n",
        "            rec = 0\n",
        "        else:\n",
        "            rec = tp / (tp + fn)\n",
        "        if (prec+rec) == 0:\n",
        "            f1 = 0\n",
        "        else:\n",
        "            f1 = (2*(prec*rec))/(prec+rec)\n",
        "            \n",
        "        jac = tp / union\n",
        "        \n",
        "        precisions.append(prec)\n",
        "        recalls.append(rec)\n",
        "        f1s.append(f1)\n",
        "        jaccards.append(jac)\n",
        "    print('Precision - ', round(np.mean(precisions), 2))\n",
        "    print('Recall - ', round(np.mean(recalls), 2))\n",
        "    print('F1 - ', round(np.mean(f1s), 2))\n",
        "    print('Jaccard - ', round(np.mean(jaccards), 2))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VpzqfOMyev0g",
        "colab_type": "code",
        "outputId": "e45af959-13b5-4997-9b26-bffc8252a3f3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "evaluate(data['keywords'], data['keywords'])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Precision -  1.0\n",
            "Recall -  1.0\n",
            "F1 -  1.0\n",
            "Jaccard -  1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jMIbjEtZe2uR",
        "colab_type": "text"
      },
      "source": [
        "### Бейзлайн"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W1GiQEote1c9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tfidf = TfidfVectorizer()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lRMd-GZIe67c",
        "colab_type": "code",
        "outputId": "4e7c4cf4-6bc1-4733-baa5-2cea484efa65",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "tfidf.fit(data['content'])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
              "                dtype=<class 'numpy.float64'>, encoding='utf-8',\n",
              "                input='content', lowercase=True, max_df=1.0, max_features=None,\n",
              "                min_df=1, ngram_range=(1, 1), norm='l2', preprocessor=None,\n",
              "                smooth_idf=True, stop_words=None, strip_accents=None,\n",
              "                sublinear_tf=False, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
              "                tokenizer=None, use_idf=True, vocabulary=None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nqoiBnYMe8fg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "id2word = {i:word for i,word in enumerate(tfidf.get_feature_names())}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bdz84nnbe-52",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "texts_vectors = tfidf.transform(data['content'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_FQQ17XyfQQg",
        "colab_type": "code",
        "outputId": "e3509b2d-dd1e-49a3-9dcb-25fa4d1bc264",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "texts_vectors.getrow(0).toarray().argsort()[0,:-11:-1]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([366589, 452726, 109689, 340186, 378920, 474377, 326570, 381182,\n",
              "       312079, 496467])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HirjXT8txKaI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "keywords = []\n",
        "\n",
        "for row in range(texts_vectors.shape[0]):\n",
        "    row_data = texts_vectors.getrow(row)\n",
        "    top_inds = row_data.toarray().argsort()[0,:-11:-1]\n",
        "    keywords.append([id2word[w] for w in top_inds])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_6eVnz3_1KUS",
        "colab_type": "code",
        "outputId": "f8355d74-024b-49f4-c488-fca572cab638",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 527
        }
      },
      "source": [
        "keywords[:3]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['игу',\n",
              "  'мион',\n",
              "  'irkutsk',\n",
              "  'дятлов',\n",
              "  'иркутск',\n",
              "  'научно',\n",
              "  'диаспоры',\n",
              "  'исследовательской',\n",
              "  'григоричев',\n",
              "  'ноц'],\n",
              " ['бурятии',\n",
              "  'бурятия',\n",
              "  'на',\n",
              "  'острогов',\n",
              "  'республики',\n",
              "  'налогу',\n",
              "  'транспортной',\n",
              "  'xvii',\n",
              "  'месторождении',\n",
              "  'монголией'],\n",
              " ['10',\n",
              "  'галактик',\n",
              "  'lk',\n",
              "  '11',\n",
              "  '12',\n",
              "  'сверхскоплении',\n",
              "  '13',\n",
              "  'sdss',\n",
              "  'пар',\n",
              "  '2masx']]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MlDMfDntfTqO",
        "colab_type": "code",
        "outputId": "05ab95a4-361a-48e9-b308-eacc92dbadc9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "evaluate(data['keywords'], keywords)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Precision -  0.05\n",
            "Recall -  0.08\n",
            "F1 -  0.06\n",
            "Jaccard -  0.03\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ROaUtHFfOGQy",
        "colab_type": "text"
      },
      "source": [
        "### 1 способ - стемминг, удаление стоп-слов, нормализация"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u27hS8PeGw_o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def normalize(text):\n",
        "    \n",
        "    words = [word.strip(punct) for word in text.lower().split()]\n",
        "    words = [stemmer.stem(word) for word in words if word and word not in stops]\n",
        "\n",
        "    return words"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tr8v-wLlZ5uX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def normalize2(text):\n",
        "    \n",
        "    words = [stemmer.stem(word) for word in text if word and word not in stops]\n",
        "\n",
        "    return words"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZTfvn7kHSonm",
        "colab_type": "code",
        "outputId": "dc79564f-cf4c-494f-b0d2-e046f00ecfe9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "%%time \n",
        "data['content_norm'] = data['content'].apply(normalize)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 14min 59s, sys: 1.35 s, total: 15min\n",
            "Wall time: 15min 1s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G-pF0dblTNbB",
        "colab_type": "code",
        "outputId": "96c1ad2c-2744-4b75-a226-96bf0ca92538",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "%%time\n",
        "evaluate(data['keywords'], data['content_norm'].apply(lambda x: [x[0] for x in Counter(x).most_common(10)]))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Precision -  0.04\n",
            "Recall -  0.06\n",
            "F1 -  0.05\n",
            "Jaccard -  0.02\n",
            "CPU times: user 4.18 s, sys: 9.99 ms, total: 4.19 s\n",
            "Wall time: 4.19 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nXMFz5jfaPOM",
        "colab_type": "code",
        "outputId": "f3192463-a2df-4a19-e70a-0b02d9bde7c8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "%%time \n",
        "data['keywords_stem'] = data['keywords'].apply(normalize2)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 7.95 s, sys: 19 ms, total: 7.96 s\n",
            "Wall time: 7.96 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uFDmv9isaz_v",
        "colab_type": "code",
        "outputId": "c2a739be-b81d-4eb7-91a9-2bcbaed5110d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "%%time\n",
        "evaluate(data['keywords_stem'], data['content_norm'].apply(lambda x: [x[0] for x in Counter(x).most_common(10)]))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Precision -  0.11\n",
            "Recall -  0.16\n",
            "F1 -  0.12\n",
            "Jaccard -  0.07\n",
            "CPU times: user 3.6 s, sys: 16 ms, total: 3.61 s\n",
            "Wall time: 3.61 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R60aT2LnbE65",
        "colab_type": "text"
      },
      "source": [
        "### Вывод:\n",
        "Использование стемминга и удаление стоп-слов как в колонке content, так и в колонке keywords позволило значительно улучшить результаты бейзлайна. Стемминг показал себя хуже, чем лемматизация, но так как я не могу применить последнюю - это неплохой вариант. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8aZ812ecOJYJ",
        "colab_type": "text"
      },
      "source": [
        "### 2 способ - настройка tfidf-векторизатора"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "msTDrUBLDlKg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def normalize(text):\n",
        "    \n",
        "    words = [word.strip(punct) for word in text.lower().split()]\n",
        "    words = [stemmer.stem(word) for word in words if word and word not in stops]\n",
        "\n",
        "    return words"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DTsHVMUYOJxC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tfidf = TfidfVectorizer(tokenizer=normalize, min_df=10, max_df=0.9) #stop_words=stops,"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zWRc8a6feAZP",
        "colab_type": "code",
        "outputId": "71b8db0c-0dc5-457a-d18e-f4e2797523d0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 207
        }
      },
      "source": [
        "tfidf.fit(data['content'])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/feature_extraction/text.py:507: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n",
            "  warnings.warn(\"The parameter 'token_pattern' will not be used\"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
              "                dtype=<class 'numpy.float64'>, encoding='utf-8',\n",
              "                input='content', lowercase=True, max_df=0.9, max_features=None,\n",
              "                min_df=10, ngram_range=(1, 1), norm='l2', preprocessor=None,\n",
              "                smooth_idf=True, stop_words=None, strip_accents=None,\n",
              "                sublinear_tf=False, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
              "                tokenizer=<function normalize at 0x7fdff5ade7b8>, use_idf=True,\n",
              "                vocabulary=None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mARs4-_3eA3A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "id2word = {i:word for i,word in enumerate(tfidf.get_feature_names())}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gw9R1xkceBQR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "texts_vectors = tfidf.transform(data['content'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3jTytnFMeagC",
        "colab_type": "code",
        "outputId": "5a7feb17-8b8b-49d3-fb5d-3951c5cef096",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "texts_vectors.getrow(0).toarray().argsort()[0,:-11:-1]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([19547, 19718, 33835,  3752, 23937, 16903, 16014, 25846, 25840,\n",
              "       30215])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PfhgPUX4eaa8",
        "colab_type": "code",
        "outputId": "f24970a0-27ca-41f4-ed23-af6cc6fdf7d3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "%%time\n",
        "keywords = []\n",
        "\n",
        "for row in range(texts_vectors.shape[0]):\n",
        "    row_data = texts_vectors.getrow(row)\n",
        "    top_inds = row_data.toarray().argsort()[0,:-11:-1]\n",
        "    keywords.append([id2word[w] for w in top_inds])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 20.4 s, sys: 171 ms, total: 20.5 s\n",
            "Wall time: 20.5 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2tY6O1FKeaPW",
        "colab_type": "code",
        "outputId": "5b980e2f-0b19-49b7-aa1a-0c4d5ca00bdb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "evaluate(data['keywords'], keywords)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Precision -  0.04\n",
            "Recall -  0.06\n",
            "F1 -  0.05\n",
            "Jaccard -  0.03\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LWyPP3pAPr8C",
        "colab_type": "code",
        "outputId": "fe38694d-064e-4000-97aa-dbd89ca8a23c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "evaluate(data['keywords_stem'], keywords)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Precision -  0.12\n",
            "Recall -  0.17\n",
            "F1 -  0.13\n",
            "Jaccard -  0.07\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d-URthDRYUpw",
        "colab_type": "text"
      },
      "source": [
        "### Вывод:\n",
        "Использование стемминга, удаление стоп-слов и tf-idf векторизация с измененными параметрами дало лучший результат из всех опробованных мною способов. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cgO7bt4aOKks",
        "colab_type": "text"
      },
      "source": [
        "### 3 способ - графы с пересекающимися окнами"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PPAPWIJPotV3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def take_step(n, matrix):\n",
        "    rang = len(matrix[n])\n",
        "    # выбираем узел из заданного интервала, на основе распределения из матрицы совстречаемости\n",
        "    if np.any(matrix[n]):\n",
        "        next_n = np.random.choice(range(rang), p=matrix[n])\n",
        "    else:\n",
        "        next_n = np.random.choice(range(rang))\n",
        "    return next_n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eANipnz-ovWx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_kws(text, top=5, random_p=0.1):\n",
        "    try:\n",
        "      vocab = set(text)\n",
        "      m = np.zeros((len(vocab), len(vocab)))\n",
        "      word2id = {w:i for i, w in enumerate(vocab)}\n",
        "      id2word = {i:w for i, w in enumerate(vocab)}\n",
        "      # преобразуем слова в индексы для удобства\n",
        "      ids = [word2id[word] for word in text]\n",
        "\n",
        "    # пройдемся окном по всему тексту\n",
        "      for i in range(0, len(ids)):\n",
        "          window = ids[i:i+15]\n",
        "          # добавим единичку всем парам слов в этом окне\n",
        "          for j, k in combinations(window, 2):\n",
        "              # чтобы граф был ненаправленный \n",
        "              m[j][k] += 1\n",
        "              m[k][j] += 1\n",
        "    \n",
        "    # нормализуем строки, чтобы получилась вероятность перехода\n",
        "      for i in range(m.shape[0]):\n",
        "          s = np.sum(m[i])\n",
        "          if not s:\n",
        "              continue\n",
        "          m[i] /= s\n",
        "    \n",
        "    # случайно выберем первое слова, а затем будет выбирать на основе полученых распределений\n",
        "    # сделаем так 5 раз и добавим каждое слово в счетчик\n",
        "    # чтобы не забиться в одном круге, иногда будет перескакивать на случайное слово\n",
        "    \n",
        "      c = Counter()\n",
        "    # начнем с абсолютного случайно выбранного элемента\n",
        "      n = np.random.choice(len(vocab))\n",
        "      for i in range(500): # если долго считается, можно уменьшить число проходов\n",
        "        \n",
        "        # c вероятностью random_p \n",
        "        # перескакиваем на другой узел\n",
        "          go_random = np.random.choice([0, 1], p=[1-random_p, random_p])\n",
        "        \n",
        "          if go_random:\n",
        "              n = np.random.choice(len(vocab))\n",
        "        \n",
        "        \n",
        "        ### \n",
        "          n = take_step(n, m)\n",
        "        # записываем узлы, в которых были\n",
        "          c.update([n])\n",
        "    \n",
        "    except ValueError:\n",
        "      pass\n",
        "    \n",
        "    # вернем топ-N наиболее часто встретившихся сл\n",
        "    return [id2word[i] for i, count in c.most_common(top)]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oDG8MzVp90JV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def normalize_lite(text):\n",
        "    \n",
        "    words = [word.strip(punct) for word in text.lower().split()]\n",
        "    words = [word for word in words if word and word not in stops]\n",
        "\n",
        "    return words"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BbVLOKkS9_6C",
        "colab_type": "code",
        "outputId": "1bea044c-b49f-4cc5-9881-672a1193ab05",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "%%time\n",
        "data['content_norm_lite'] = data['content'].apply(normalize_lite)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 11.5 s, sys: 1.96 s, total: 13.5 s\n",
            "Wall time: 13.5 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rB7V75vPVW_7",
        "colab_type": "code",
        "outputId": "4ca3b138-8a09-4c9e-97d1-e8b6a6b80428",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "data['content_norm_lite'].head(5)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    [этнополитические, миграционные, диаспоральные, процессы, переселенческом, обществе, ethnopolitical, migratory, and, diaspora's, processes, in, the, resettlement, society, серия, политология, религиоведение, 2012, 2, 9, ч, 2, 5–12, онлайн-доступ, журналу, http://isu.ru/izvestia, известия, иркутского, государственного, университета, удк, 314.7, исследовательского, центра, исследовательской, сети, этномиграционные, диаспоральные, исследования, ноц, мион, игу, григоричев, дятлов, научно-образовательный, центр, межрегиональный, институт, общественных, наук, иркутском, государственном, университете, иркутский, государственный, университет, г, иркутск, данная, статья, посвящена, развитию, этномиграционных, диаспоральных, исследований, научно-образовательном, центре, мион, игу, авторы, описывают, основные, этапы, формирования, межрегионального, междисциплинарного, коллектива, последние, 20, лет, основные, результаты, исследовательской, работы, ноц, мион, игу, перспективы, специальная, час...\n",
              "1                                                                              [2011’05, власть, 5, âÿ÷åñëàâ, íàãîâèöûí, 350, ëåò, äîáðîâîëüíîãî, âõîæäåíèß, áóðßòèè, â, ñîñòàâ, ðîññèéñêîãî, ãîñóäàðñòâà, â, ñòàòüå, ðàññìàòðèâàåòñÿ, ñîâðåìåííîå, ñîñòîÿíèå, è, ïåðñïåêòèâû, ðàçâèòèÿ, ðåñïóáëèêè, áóðÿòèÿ, êàê, ñëåäñòâèå, äîáðîâîëüíîãî, âõîæäåíèÿ, â, ñîñòàâ, ðîññèéñêîãî, ãîñóäàðñòâà, the, article, considers, contemporary, condition, and, perspectives, of, development, of, the, republic, of, buryatia, as, a, result, of, the, voluntary, inclusion, into, the, russian, state, наговицын, вячеслав, владимирович, президент, председатель, правительства, республики, бурятия, druzh@govrb.ru, июне, 2011, г, исполняется, 350, лет, добровольного, вхождения, бурятии, состав, российского, государства, 20-е, гг, xvii, первые, немногочисленные, отряды, русских, служилых, людей, главе, атаманами, м, перфильевым, хрипуновым, прибывают, предбайкалье, собрав, необходимые, сведения, получив, согласие, бурятских, ...]\n",
              "2                                                                                                                                        [астрофизический, бюллетень, 2008, 63, 4, 320–368, удк, 524.74:524.77, двойные, галактики, местном, сверхскоплении, окрестностях, c, 2008, \u0001, 1, д, караченцев1, д, макаров1, 2, специальная, астрофизическая, обсерватория, нижний, архыз, 369167, россия1, 2, invited, researcher, observatoire, de, lyon, st-genis, laval, cedex, france, 69561, поступила, редакцию, 22, мая, 2008, г, принята, печать, 9, июня, 2008, г, представляем, каталог, 509, пар, выделенных, среди, 10403, близких, галактик, лучевыми, скоростями, vlg, 3500, км/с, двойные, системы, отобраны, двум, критериям, связывающий, временной, которые, требуют, физическая, пара, галактик, имела, отрицательную, полную, энергию, компоненты, находились, внутри, поверхности, нулевой, скорости, индивидуальные, массы, галактик, предполагаются, пропорциональными, полной, светимости, k-фильтре, m, lk, ×, 6m\u0001, ...]\n",
              "3                                                                                                                 [issn, 0321–3056, известия, вузов, северо-кавказский, регион, общественные, науки, 2008, 4, ббк, э210.012+ю711, движение, нью, эйдж, трансформация, общественного, сознания, ©, 2008, г, е.п, агапов, л.п, пендюрина, южный, федеральный, университет, 344038, ростов-на-дону, пр, м, нагибина, 13, march10@mail.ru, southern, federal, university, 344038, rostov-on-don, m, nagibin, ave, 13, march10@mail.ru, изучено, движение, нью, эйдж, сформировавшееся, западной, культуре, начале, 60-х, гг, хх, анализируются, взгляды, м, фергюсон, идеолога, движения, сумевшего, отразить, изменения, сознании, ментальности, умонастроениях, представителей, западной, культуры, установлена, степень, влияния, движения, нью, эйдж, развитие, общественного, сознания, странах, запада, также, роль, современных, условиях, the, article, is, devoted, to, the, new, age, movement, which, emerged, in, the, western, ...]\n",
              "4      [м, тулякова, развитие, самодеятельного, творчества, малых, городах, горьковской, области, условиях, культурной, политики, 40–60-х, гг, xx, статье, анализируются, проблемы, послевоенного, развития, культурно-просветительных, учреждений, горьковской, области, партийное, руководство, соответствии, идеологическими, установками, осуществляло, мероприятия, направленные, улучшение, деятельности, коллективов, художественной, самодеятельности, анализ, архивных, источников, свидетельствует, реализация, общегосударственных, задач, развитию, художественной, самодеятельности, проводилась, учета, специфики, малых, городов, горьковской, области, m, tuliakova, development, of, amateur, arts, in, the, small, towns, of, gorky, region, in, the, context, of, cultural, policy, in, the, 1940–1960s, the, issues, of, postwar, development, of, cultural, and, educational, institutions, in, gorki, region, are, regarded, the, party, authorities, according, to, ideological, goals, implemented, activities, ...]\n",
              "Name: content_norm_lite, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4M7uKZXOyTwU",
        "colab_type": "code",
        "outputId": "2cee4608-409b-4526-94f3-ef1aa3208ca5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "%%time\n",
        "for i, line in enumerate(data['keywords']):\n",
        "  for j, exp in enumerate(line):\n",
        "    #print(i, j, exp.lower())\n",
        "    word = data['keywords'][i][j]\n",
        "    low_word = word.lower()\n",
        "    data['keywords'][i][j] = low_word"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 4.06 s, sys: 0 ns, total: 4.06 s\n",
            "Wall time: 4.06 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pE6XI4JveLrc",
        "colab_type": "text"
      },
      "source": [
        "### Окно из 5 слов"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-2Gq839GCY45",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "keywords_rw = data['content_norm_lite'].apply(lambda x: get_kws(x))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EfdW2J_OW7Rr",
        "colab_type": "code",
        "outputId": "a1a0eb04-4df5-4a95-d6fe-da2851418986",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "evaluate(data['keywords'], keywords_rw) "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Precision -  0.06\n",
            "Recall -  0.04\n",
            "F1 -  0.05\n",
            "Jaccard -  0.03\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QKd5N-CaePAN",
        "colab_type": "text"
      },
      "source": [
        "### Окно из 10 слов"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4dKXy2gHeSap",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "keywords_rw = data['content_norm_lite'].apply(lambda x: get_kws(x))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yna8MqnYeSog",
        "colab_type": "code",
        "outputId": "77cd5a6e-ef91-4292-b029-7f77dc0f1dad",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "evaluate(data['keywords'], keywords_rw)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Precision -  0.06\n",
            "Recall -  0.05\n",
            "F1 -  0.05\n",
            "Jaccard -  0.03\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FHmSiPuBxXok",
        "colab_type": "text"
      },
      "source": [
        "### Окно из 3 слов"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nszZFmFQxYGw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "keywords_rw = data['content_norm_lite'].apply(lambda x: get_kws(x))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T_X50XM8xYT6",
        "colab_type": "code",
        "outputId": "9e7a02c0-9f64-41f3-f705-9bfd9890b444",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "evaluate(data['keywords'], keywords_rw)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Precision -  0.05\n",
            "Recall -  0.04\n",
            "F1 -  0.04\n",
            "Jaccard -  0.02\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gWmjv3om7L-S",
        "colab_type": "text"
      },
      "source": [
        "### Окно из 15 слов"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4D_sLm5L8aui",
        "colab_type": "code",
        "outputId": "df39e701-d8fa-4c44-cfe1-00e1ef3ef941",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "%%time \n",
        "keywords_rw = data['content_norm_lite'].apply(lambda x: get_kws(x))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 1h 8min 12s, sys: 15.8 s, total: 1h 8min 28s\n",
            "Wall time: 1h 8min 28s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kMvb756Y7M2x",
        "colab_type": "code",
        "outputId": "3f4285d8-907b-475a-9fe8-e85f55a0198b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "evaluate(data['keywords'], keywords_rw)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Precision -  0.06\n",
            "Recall -  0.05\n",
            "F1 -  0.05\n",
            "Jaccard -  0.03\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f-qCA3pmZ5l5",
        "colab_type": "text"
      },
      "source": [
        "### Вывод:\n",
        "Использование факторов с пересекающимися окнами позволило улучшить результат бейзлайна Precision на 0.1. Остальные метрики немного хуже. Возможно получилось бы значительно улучшить результат, использовав лемматизацию Pymorphy, но на таком количестве данных google colab и jupyter на windows не позволяют этого сделать."
      ]
    }
  ]
}