WSD and WSI.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "HW7.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "aAb1_VKZ9jIB",
        "colab_type": "code",
        "outputId": "e09a269f-6e97-4615-8444-182922d2dd4e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 286
        }
      },
      "source": [
        "!pip install Cython numpy\n",
        "!pip install git+https://github.com/lopuhin/python-adagram.git"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: Cython in /usr/local/lib/python3.6/dist-packages (0.29.15)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (1.17.5)\n",
            "Collecting git+https://github.com/lopuhin/python-adagram.git\n",
            "  Cloning https://github.com/lopuhin/python-adagram.git to /tmp/pip-req-build-svszo4hd\n",
            "  Running command git clone -q https://github.com/lopuhin/python-adagram.git /tmp/pip-req-build-svszo4hd\n",
            "Requirement already satisfied: cython in /usr/local/lib/python3.6/dist-packages (from adagram==0.0.1) (0.29.15)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from adagram==0.0.1) (0.14.1)\n",
            "Requirement already satisfied: numpy>=1.9 in /usr/local/lib/python3.6/dist-packages (from adagram==0.0.1) (1.17.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from adagram==0.0.1) (1.12.0)\n",
            "Building wheels for collected packages: adagram\n",
            "  Building wheel for adagram (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for adagram: filename=adagram-0.0.1-cp36-cp36m-linux_x86_64.whl size=464611 sha256=6bed2f4fba64a1807b0914ea3b112aaa8ca75c8ab05a49507311037485fef773\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-y7m8x9qr/wheels/11/0f/46/f5df96670df8f7973b4c2311ffc9b02e435a7bd3207f992c4d\n",
            "Successfully built adagram\n",
            "Installing collected packages: adagram\n",
            "Successfully installed adagram-0.0.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "flycoHN6-DEu",
        "colab_type": "code",
        "outputId": "bbd90b90-7945-4843-cb0c-d2f57dbd5f27",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 222
        }
      },
      "source": [
        "!pip install pymorphy2"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pymorphy2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a3/33/fff9675c68b5f6c63ec8c6e6ff57827dda28a1fa5b2c2d727dffff92dd47/pymorphy2-0.8-py2.py3-none-any.whl (46kB)\n",
            "\r\u001b[K     |███████                         | 10kB 19.1MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 20kB 3.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 30kB 3.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 40kB 3.0MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 51kB 2.7MB/s \n",
            "\u001b[?25hCollecting dawg-python>=0.7\n",
            "  Downloading https://files.pythonhosted.org/packages/6a/84/ff1ce2071d4c650ec85745766c0047ccc3b5036f1d03559fd46bb38b5eeb/DAWG_Python-0.7.2-py2.py3-none-any.whl\n",
            "Collecting pymorphy2-dicts<3.0,>=2.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/02/51/2465fd4f72328ab50877b54777764d928da8cb15b74e2680fc1bd8cb3173/pymorphy2_dicts-2.4.393442.3710985-py2.py3-none-any.whl (7.1MB)\n",
            "\u001b[K     |████████████████████████████████| 7.1MB 7.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: docopt>=0.6 in /usr/local/lib/python3.6/dist-packages (from pymorphy2) (0.6.2)\n",
            "Installing collected packages: dawg-python, pymorphy2-dicts, pymorphy2\n",
            "Successfully installed dawg-python-0.7.2 pymorphy2-0.8 pymorphy2-dicts-2.4.393442.3710985\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uLPAbGUM-JGu",
        "colab_type": "code",
        "outputId": "bba57db3-d488-483a-9e49-2e8077d96bea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101
        }
      },
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yPuoOunE-Ab5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import adagram\n",
        "from lxml import html\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.corpus import wordnet as wn\n",
        "from pymorphy2 import MorphAnalyzer\n",
        "from string import punctuation\n",
        "import json, os\n",
        "from collections import Counter\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from matplotlib import pyplot as plt\n",
        "%matplotlib inline\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "morph = MorphAnalyzer()\n",
        "punct = punctuation+'«»—…“”*№–'\n",
        "stops = set(stopwords.words('russian'))\n",
        "\n",
        "\n",
        "\n",
        "def tokenize(text):\n",
        "    \n",
        "    words = [word.strip(punct) for word in text.lower().split() if word and word not in stops]\n",
        "    words = [word for word in words if word]\n",
        "\n",
        "    return words\n",
        "\n",
        "def normalize(text):\n",
        "    \n",
        "    words = tokenize(text)\n",
        "    words = [morph.parse(word)[0].normal_form for word in words if word]\n",
        "\n",
        "    return words"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LyjhjibXAQkl",
        "colab_type": "code",
        "outputId": "466b8ca0-0fc3-4abd-9349-9fe0059dc2ea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 138
        }
      },
      "source": [
        "from google.colab import drive \n",
        "drive.mount('/gdrive')\n",
        "%cd /gdrive\n"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /gdrive\n",
            "/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dXF_PB94ETCH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vm = adagram.VectorModel.load(\"/gdrive/My Drive/WSD_WSI/out.pkl\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "29mRsFiD-502",
        "colab_type": "text"
      },
      "source": [
        "Задание № 1. Протестировать адаграм в определении перефразирования"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o7e2yZ_6B2zR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "para = '/gdrive/My Drive/WSD_WSI/paraphraser/paraphrases.xml'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rv3jAG3A-MIL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "corpus_xml = html.fromstring(open(para, 'rb').read())\n",
        "texts_1 = []\n",
        "texts_2 = []\n",
        "classes = []\n",
        "\n",
        "for p in corpus_xml.xpath('//paraphrase'):\n",
        "    texts_1.append(p.xpath('./value[@name=\"text_1\"]/text()')[0])\n",
        "    texts_2.append(p.xpath('./value[@name=\"text_2\"]/text()')[0])\n",
        "    classes.append(p.xpath('./value[@name=\"class\"]/text()')[0])\n",
        "    \n",
        "data = pd.DataFrame({'text_1':texts_1, 'text_2':texts_2, 'label':classes})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "txFHCowsDLXc",
        "colab_type": "code",
        "outputId": "892ac44e-2a18-4a08-d580-4e2feffce950",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        }
      },
      "source": [
        "data"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text_1</th>\n",
              "      <th>text_2</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Полицейским разрешат стрелять на поражение по ...</td>\n",
              "      <td>Полиции могут разрешить стрелять по хулиганам ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Право полицейских на проникновение в жилище ре...</td>\n",
              "      <td>Правила внесудебного проникновения полицейских...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Президент Египта ввел чрезвычайное положение в...</td>\n",
              "      <td>Власти Египта угрожают ввести в стране чрезвыч...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Вернувшихся из Сирии россиян волнует вопрос тр...</td>\n",
              "      <td>Самолеты МЧС вывезут россиян из разрушенной Си...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>В Москву из Сирии вернулись 2 самолета МЧС с р...</td>\n",
              "      <td>Самолеты МЧС вывезут россиян из разрушенной Си...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7222</th>\n",
              "      <td>Путин освободил от должности ряд генералов</td>\n",
              "      <td>Путин снял с должностей более 20 руководителей...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7223</th>\n",
              "      <td>Облака над Москвой в День Победы разгонят девя...</td>\n",
              "      <td>Путеводитель по Дню Победы: как провести 9 мая...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7224</th>\n",
              "      <td>Любляна отпразднует День Победы вместе с Москвой</td>\n",
              "      <td>В Москве ограничат движение в связи с Днем Победы</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7225</th>\n",
              "      <td>Девять самолетов ВВС разгонят облака над Москв...</td>\n",
              "      <td>В Москве ограничат движение в связи с Днем Победы</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7226</th>\n",
              "      <td>9 мая метрополитен Петербурга будет работать к...</td>\n",
              "      <td>Мартынов: комендантский час в Донецке 9 мая бу...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>7227 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                 text_1  ... label\n",
              "0     Полицейским разрешат стрелять на поражение по ...  ...     0\n",
              "1     Право полицейских на проникновение в жилище ре...  ...     0\n",
              "2     Президент Египта ввел чрезвычайное положение в...  ...     0\n",
              "3     Вернувшихся из Сирии россиян волнует вопрос тр...  ...    -1\n",
              "4     В Москву из Сирии вернулись 2 самолета МЧС с р...  ...     0\n",
              "...                                                 ...  ...   ...\n",
              "7222         Путин освободил от должности ряд генералов  ...     0\n",
              "7223  Облака над Москвой в День Победы разгонят девя...  ...    -1\n",
              "7224   Любляна отпразднует День Победы вместе с Москвой  ...    -1\n",
              "7225  Девять самолетов ВВС разгонят облака над Москв...  ...    -1\n",
              "7226  9 мая метрополитен Петербурга будет работать к...  ...    -1\n",
              "\n",
              "[7227 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j2M5OqmpDQ09",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data['text_1_norm'] = data['text_1'].apply(normalize)\n",
        "data['text_2_norm'] = data['text_2'].apply(normalize)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-_YYiKodFJsa",
        "colab_type": "code",
        "outputId": "be98ebf8-e33e-4395-b5ca-1ecb8e758c03",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        }
      },
      "source": [
        "data.head(5)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text_1</th>\n",
              "      <th>text_2</th>\n",
              "      <th>label</th>\n",
              "      <th>text_1_norm</th>\n",
              "      <th>text_2_norm</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Полицейским разрешат стрелять на поражение по ...</td>\n",
              "      <td>Полиции могут разрешить стрелять по хулиганам ...</td>\n",
              "      <td>0</td>\n",
              "      <td>[полицейский, разрешить, стрелять, поражение, ...</td>\n",
              "      <td>[полиция, мочь, разрешить, стрелять, хулиган, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Право полицейских на проникновение в жилище ре...</td>\n",
              "      <td>Правила внесудебного проникновения полицейских...</td>\n",
              "      <td>0</td>\n",
              "      <td>[право, полицейский, проникновение, жилища, ре...</td>\n",
              "      <td>[правило, внесудебный, проникновение, полицейс...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Президент Египта ввел чрезвычайное положение в...</td>\n",
              "      <td>Власти Египта угрожают ввести в стране чрезвыч...</td>\n",
              "      <td>0</td>\n",
              "      <td>[президент, египет, ввести, чрезвычайный, поло...</td>\n",
              "      <td>[власть, египет, угрожать, ввести, страна, чре...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Вернувшихся из Сирии россиян волнует вопрос тр...</td>\n",
              "      <td>Самолеты МЧС вывезут россиян из разрушенной Си...</td>\n",
              "      <td>-1</td>\n",
              "      <td>[вернуться, сирия, россиянин, волновать, вопро...</td>\n",
              "      <td>[самолёт, мчс, вывезти, россиянин, разрушить, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>В Москву из Сирии вернулись 2 самолета МЧС с р...</td>\n",
              "      <td>Самолеты МЧС вывезут россиян из разрушенной Си...</td>\n",
              "      <td>0</td>\n",
              "      <td>[москва, сирия, вернуться, 2, самолёт, мчс, ро...</td>\n",
              "      <td>[самолёт, мчс, вывезти, россиянин, разрушить, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                              text_1  ...                                        text_2_norm\n",
              "0  Полицейским разрешат стрелять на поражение по ...  ...  [полиция, мочь, разрешить, стрелять, хулиган, ...\n",
              "1  Право полицейских на проникновение в жилище ре...  ...  [правило, внесудебный, проникновение, полицейс...\n",
              "2  Президент Египта ввел чрезвычайное положение в...  ...  [власть, египет, угрожать, ввести, страна, чре...\n",
              "3  Вернувшихся из Сирии россиян волнует вопрос тр...  ...  [самолёт, мчс, вывезти, россиянин, разрушить, ...\n",
              "4  В Москву из Сирии вернулись 2 самолета МЧС с р...  ...  [самолёт, мчс, вывезти, россиянин, разрушить, ...\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wx8rEWVB1juX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_words_in_context(words, n=3):\n",
        "  neighbours = [] \n",
        "  for i in range(len(words)):\n",
        "    if i-n<0:\n",
        "      words_after = words[i+1:i+n+1]\n",
        "      words_before = words[0:i]\n",
        "      ws = words_before + words_after\n",
        "      lst = list([words[i], ws])\n",
        "      neighbours.append(lst)   \n",
        "\n",
        "    else:\n",
        "      words_after = words[i+1:i+n+1]\n",
        "      words_before = words[i-n:i]\n",
        "      ws = words_before + words_after\n",
        "      lst = list([words[i], ws])\n",
        "      neighbours.append(lst)\n",
        "\n",
        "  return neighbours"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m6dOf_DWRL7l",
        "colab_type": "code",
        "outputId": "c1db5c9d-bfe1-43ba-efde-48f9f9187be8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 185
        }
      },
      "source": [
        "words = [0,1,2,3,4,5,6,7,8,9]\n",
        "get_words_in_context(words, n=3)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[0, [1, 2, 3]],\n",
              " [1, [0, 2, 3, 4]],\n",
              " [2, [0, 1, 3, 4, 5]],\n",
              " [3, [0, 1, 2, 4, 5, 6]],\n",
              " [4, [1, 2, 3, 5, 6, 7]],\n",
              " [5, [2, 3, 4, 6, 7, 8]],\n",
              " [6, [3, 4, 5, 7, 8, 9]],\n",
              " [7, [4, 5, 6, 8, 9]],\n",
              " [8, [5, 6, 7, 9]],\n",
              " [9, [6, 7, 8]]]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ImnIY0--DwL0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_embedding_adagram(text, model, n, dim):\n",
        "    \n",
        "    word2context = get_words_in_context(text, n)\n",
        "    \n",
        "    \n",
        "    vectors = np.zeros((len(word2context), dim))\n",
        "    \n",
        "    for i, (word, context) in enumerate(word2context):\n",
        "        \n",
        "        try:\n",
        "            v = model.sense_vector(word, vm.disambiguate(word, context).argmax())\n",
        "            vectors[i] = v\n",
        "        \n",
        "        except (KeyError, ValueError):\n",
        "            continue\n",
        "    \n",
        "    if vectors.any():\n",
        "        vector = np.average(vectors, axis=0)\n",
        "    else:\n",
        "        vector = np.zeros((dim))\n",
        "    \n",
        "    return vector\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zvad9g1cOZK7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dim=100\n",
        "X_text_1 = np.zeros((len(data['text_1_norm']), dim))\n",
        "X_text_2 = np.zeros((len(data['text_2_norm']), dim))\n",
        "\n",
        "for i, text in enumerate(data['text_1_norm'].values):\n",
        "    X_text_1[i] = get_embedding_adagram(text, vm, n=3, dim=100)\n",
        "    \n",
        "for i, text in enumerate(data['text_2_norm'].values):\n",
        "    X_text_2[i] = get_embedding_adagram(text, vm, n=3, dim=100)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JSyPu9QWFf2o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_text = np.concatenate([X_text_1, X_text_2], axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W_GPuyHUEcsi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn import svm\n",
        "from sklearn import metrics\n",
        "from sklearn.model_selection import cross_val_score"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8R0H-qyCE-76",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = data['label'].values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lgxzaVOTFNXO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "clf = svm.SVC(kernel='linear', C=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mOYwIoeDFROW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "scores = cross_val_score(clf, X_text, y, cv=5, scoring='f1_macro')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c3LpewQFGTvY",
        "colab_type": "code",
        "outputId": "32d29a4e-d94b-405e-8dcc-bec6c8870510",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "scores"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.32923635, 0.33564626, 0.32806947, 0.28493003, 0.28372589])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zGkyB2tdGZjT",
        "colab_type": "code",
        "outputId": "5fc8a939-bc75-488e-898f-b6a7a1e51a4e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "scores.mean()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.312321599532449"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4RqgWpYgG0RH",
        "colab_type": "text"
      },
      "source": [
        "Задание 2. Реализовать алгоритм Леска и проверить его на реальном датасете"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yiZiMDGRG2EN",
        "colab_type": "code",
        "outputId": "d27548c2-a190-44db-f983-c4f28ff4cc04",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "nltk.download('wordnet')"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aLMWd-wulBWp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def lesk(word, sentence):\n",
        "    bestsense = 0\n",
        "    maxoverlap = 0\n",
        "    synsets = wn.synsets(word)\n",
        "    \n",
        "    for i, syns in enumerate(synsets):\n",
        "        senses = word_tokenize(syns.definition())\n",
        "        overlap = len(set(senses).intersection(set(sentence)))\n",
        "        if overlap > maxoverlap:\n",
        "          maxoverlap = overlap\n",
        "          bestsense = i\n",
        "        \n",
        "    return bestsense\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ksqFSuaqnDvX",
        "colab_type": "code",
        "outputId": "baec3adc-9d98-4228-bd5f-02bb9092f083",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "lesk('day', 'some point or period in time'.split())"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ATSpqoDu2pu9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "corpus_wsd = []\n",
        "corpus = open('/gdrive/My Drive/WSD_WSI/corpus_wsd_50k.txt').read().split('\\n\\n')\n",
        "for sent in corpus:\n",
        "    corpus_wsd.append([s.split('\\t') for s in sent.split('\\n')])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SnXm6AXn24Rp",
        "colab_type": "code",
        "outputId": "ff891a4f-739b-4b18-fcd9-df1111a17e2d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(corpus_wsd)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "49453"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oJiGJqob3YtU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "corpus_wsd_short = corpus_wsd[:1000]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P5HHIdbwKmy-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "wordnet_list = [] #список для значений ворднета\n",
        "lesk_list = [] #список для значений леска\n",
        "\n",
        "for i, sent in enumerate(corpus_wsd_short):\n",
        "    if sent[0][0]:\n",
        "      context = []\n",
        "\n",
        "      for w in sent:\n",
        "        if '%' in w[0]:\n",
        "          context.append(w[1])\n",
        "          wn_var = wn.lemma_from_key(w[0]).synset()\n",
        "          wordnet_list.append(wn_var)\n",
        "      \n",
        "      words_in_context = get_words_in_context(context, n=3)\n",
        "      \n",
        "      for st in words_in_context:\n",
        "        i = lesk(st[0], st[1])\n",
        "        lesk_var = wn.synsets(st[0])[i]\n",
        "        lesk_list.append(lesk_var)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q7y2RFs-Q9m4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "d = {'wordnet': wordnet_list, 'lesk': lesk_list}\n",
        "df = pd.DataFrame(data=d)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SGCke5-iRZCP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 343
        },
        "outputId": "f8c4efcf-b6aa-49ab-83c0-09182f67ced5"
      },
      "source": [
        "df.head(10)"
      ],
      "execution_count": 129,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>wordnet</th>\n",
              "      <th>lesk</th>\n",
              "      <th>match</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Synset('be.v.01')</td>\n",
              "      <td>Synset('beryllium.n.01')</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Synset('bigger.s.01')</td>\n",
              "      <td>Synset('bigger.s.01')</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Synset('fancy.a.01')</td>\n",
              "      <td>Synset('fancy.n.02')</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Synset('truly.r.01')</td>\n",
              "      <td>Synset('truly.r.01')</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Synset('want.v.02')</td>\n",
              "      <td>Synset('need.n.01')</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Synset('exist.v.01')</td>\n",
              "      <td>Synset('beryllium.n.01')</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Synset('other.a.01')</td>\n",
              "      <td>Synset('other.a.01')</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Synset('cheap.a.01')</td>\n",
              "      <td>Synset('cheap.a.01')</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Synset('communication.n.01')</td>\n",
              "      <td>Synset('communication.n.01')</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Synset('technique.n.01')</td>\n",
              "      <td>Synset('technique.n.01')</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                        wordnet                          lesk  match\n",
              "0             Synset('be.v.01')      Synset('beryllium.n.01')    0.0\n",
              "1         Synset('bigger.s.01')         Synset('bigger.s.01')    1.0\n",
              "2          Synset('fancy.a.01')          Synset('fancy.n.02')    0.0\n",
              "3          Synset('truly.r.01')          Synset('truly.r.01')    1.0\n",
              "4           Synset('want.v.02')           Synset('need.n.01')    0.0\n",
              "5          Synset('exist.v.01')      Synset('beryllium.n.01')    0.0\n",
              "6          Synset('other.a.01')          Synset('other.a.01')    1.0\n",
              "7          Synset('cheap.a.01')          Synset('cheap.a.01')    1.0\n",
              "8  Synset('communication.n.01')  Synset('communication.n.01')    1.0\n",
              "9      Synset('technique.n.01')      Synset('technique.n.01')    1.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 129
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mh55KH1lRZey",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "match = np.zeros(df.shape[0])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bqw_6Lk7RtQ7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for i, syn in enumerate(df['wordnet'].values):\n",
        "  if df['lesk'][i] == syn:\n",
        "    match[i] = 1\n",
        "  else:\n",
        "    match[i] = 0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mqg6r0oeSNGA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df['match'] = match"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lM8seushSYE3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 343
        },
        "outputId": "1c5ca533-1508-4b84-f832-2aa293b1bd37"
      },
      "source": [
        "df.head(10)"
      ],
      "execution_count": 133,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>wordnet</th>\n",
              "      <th>lesk</th>\n",
              "      <th>match</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Synset('be.v.01')</td>\n",
              "      <td>Synset('beryllium.n.01')</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Synset('bigger.s.01')</td>\n",
              "      <td>Synset('bigger.s.01')</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Synset('fancy.a.01')</td>\n",
              "      <td>Synset('fancy.n.02')</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Synset('truly.r.01')</td>\n",
              "      <td>Synset('truly.r.01')</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Synset('want.v.02')</td>\n",
              "      <td>Synset('need.n.01')</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Synset('exist.v.01')</td>\n",
              "      <td>Synset('beryllium.n.01')</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Synset('other.a.01')</td>\n",
              "      <td>Synset('other.a.01')</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Synset('cheap.a.01')</td>\n",
              "      <td>Synset('cheap.a.01')</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Synset('communication.n.01')</td>\n",
              "      <td>Synset('communication.n.01')</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Synset('technique.n.01')</td>\n",
              "      <td>Synset('technique.n.01')</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                        wordnet                          lesk  match\n",
              "0             Synset('be.v.01')      Synset('beryllium.n.01')    0.0\n",
              "1         Synset('bigger.s.01')         Synset('bigger.s.01')    1.0\n",
              "2          Synset('fancy.a.01')          Synset('fancy.n.02')    0.0\n",
              "3          Synset('truly.r.01')          Synset('truly.r.01')    1.0\n",
              "4           Synset('want.v.02')           Synset('need.n.01')    0.0\n",
              "5          Synset('exist.v.01')      Synset('beryllium.n.01')    0.0\n",
              "6          Synset('other.a.01')          Synset('other.a.01')    1.0\n",
              "7          Synset('cheap.a.01')          Synset('cheap.a.01')    1.0\n",
              "8  Synset('communication.n.01')  Synset('communication.n.01')    1.0\n",
              "9      Synset('technique.n.01')      Synset('technique.n.01')    1.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 133
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qyh1kA1lSZGj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "6d536329-2907-44c1-e887-4240a34c4815"
      },
      "source": [
        "success = df[df['match'] == 1].shape[0]\n",
        "print(success)"
      ],
      "execution_count": 134,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1430\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j-Jq_ASFTme3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "97247d09-08f8-4420-c834-a79e2a117356"
      },
      "source": [
        "all_vals = df['match'].shape[0]\n",
        "print(all_vals)"
      ],
      "execution_count": 135,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2641\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kbybgKp-Tqb4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "536241db-b95f-4f13-ee72-cb7746b1e612"
      },
      "source": [
        "accuracy = success / all_vals\n",
        "print(accuracy)"
      ],
      "execution_count": 136,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.5414615675880349\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}