HW 3(1).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "HW 3(1).ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/SvA1/Compling/blob/master/HW%203(1).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YPgRBVrboF0S",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget https://raw.githubusercontent.com/mannefedov/compling_nlp_hse_course/master/data/correct_sents.txt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PmJuQEsHoiaq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget https://raw.githubusercontent.com/mannefedov/compling_nlp_hse_course/master/data/sents_with_mistakes.txt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2xcxHAtGok3_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget https://github.com/yutkin/Lenta.Ru-News-Dataset/releases/download/v1.0/lenta-ru-news.csv.gz"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sHqIxXz3ooOH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import re\n",
        "import gzip, csv\n",
        "from string import punctuation\n",
        "punctuation += \"«»—…“”\"\n",
        "punct = set(punctuation)\n",
        "from collections import Counter, defaultdict\n",
        "from pprint import pprint\n",
        "from nltk import sent_tokenize\n",
        "\n",
        "!pip install symspellpy\n",
        "from symspellpy import SymSpell, Verbosity\n",
        "import pkg_resources"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JEYgutwVo6A1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "corpus = open('corpus_5000.txt', 'w')\n",
        "with gzip.open('lenta-ru-news.csv.gz', 'rt') as archive:\n",
        "    reader = csv.reader(archive, delimiter=',', quotechar='\"')\n",
        "    for i, line in enumerate(reader):\n",
        "        if i < 5000: \n",
        "            corpus.write(line[2].replace('\\xa0', ' ') + '\\n')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jP4xOMSVpKoP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def normalize(text):\n",
        "    \n",
        "    normalized_text = [(word.strip(punctuation)) for word \\\n",
        "                                                            in text.lower().split()]\n",
        "    normalized_text = [word for word in normalized_text if word]\n",
        "    return normalized_text"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HaGyPEZUpMF7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import nltk\n",
        "nltk.download('punkt')\n",
        "corpus = []\n",
        "for text in open('corpus_5000.txt').read().splitlines()[:12000]:\n",
        "    sents = sent_tokenize(text)\n",
        "    norm_sents = [normalize(sent) for sent in sents]\n",
        "    corpus += norm_sents"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "15Wd95W4rpVs",
        "colab_type": "code",
        "outputId": "eb7b1343-529b-4d9b-e70e-e5236da90c32",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "bad = open('sents_with_mistakes.txt', encoding='utf8').read().splitlines()\n",
        "true = open('correct_sents.txt', encoding='utf8').read().splitlines()\n",
        "print(bad[1])\n",
        "print(true[1])"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Опофеозом дня для меня сегодня стала фраза услышанная в новостях:\n",
            "Апофеозом дня для меня сегодня стала фраза услышанная в новостях\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sJwtTThLvaAr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def ngrammer(tokens, n=2):\n",
        "    ngrams = []\n",
        "    for i in range(0,len(tokens)-n+1):\n",
        "        ngrams.append(' '.join(tokens[i:i+n]))\n",
        "    return ngrams"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xEUa43d22ET_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "unigrams = Counter()\n",
        "for sentence in corpus:\n",
        "    unigrams.update(sentence)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bqLVHxkeska7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def deletes(word):\n",
        "    letters    = 'йцукенгшщзхъфывапролджэячсмитьбюё'\n",
        "    splits     = [(word[:i], word[i:])     for i in range(len(word) + 1)]\n",
        "    deletes1    = [L + R[1:]               for L, R in splits if R]\n",
        "    splits2     = [(word[:i], word[i:])    for i in range(len(deletes1))]\n",
        "    deletes2    = [L + R[2:]               for L, R in splits2 if R]\n",
        "    deletes = deletes1 + deletes2\n",
        "    return set(deletes)\n",
        "    print(\"\\n\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uxjBuHTD2U0u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "glob_dict = {}\n",
        "for i in unigrams.keys():\n",
        "  for j in deletes(i):\n",
        "    glob_dict[j] = i"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3fv5qLLJo6lN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def align_words(sent_1, sent_2):\n",
        "    tokens_1 = sent_1.lower().split()\n",
        "    tokens_2 = sent_2.lower().split()\n",
        "    \n",
        "    tokens_1 = [re.sub('(^\\W+|\\W+$)', '', token) for token in tokens_1 if (set(token)-punct)]\n",
        "    tokens_2 = [re.sub('(^\\W+|\\W+$)', '', token) for token in tokens_2 if (set(token)-punct)]\n",
        "    \n",
        "    return list(zip(tokens_1, tokens_2))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MfMmu-Y9r0r-",
        "colab_type": "code",
        "outputId": "4d73d205-2b9c-434e-f7e7-186d8f798604",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196
        }
      },
      "source": [
        "pprint(align_words(true[1], bad[1]))"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('апофеозом', 'опофеозом'),\n",
            " ('дня', 'дня'),\n",
            " ('для', 'для'),\n",
            " ('меня', 'меня'),\n",
            " ('сегодня', 'сегодня'),\n",
            " ('стала', 'стала'),\n",
            " ('фраза', 'фраза'),\n",
            " ('услышанная', 'услышанная'),\n",
            " ('в', 'в'),\n",
            " ('новостях', 'новостях')]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QraLMG7elnJP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "correct = 0\n",
        "total = 0\n",
        "\n",
        "total_mistaken = 0\n",
        "mistaken_fixed = 0\n",
        "\n",
        "total_correct = 0\n",
        "correct_broken = 0\n",
        "\n",
        "cashed = {}\n",
        "for i in range(len(true)):\n",
        "    word_pairs = align_words(true[i], bad[i])\n",
        "    for pair in word_pairs:\n",
        "        predicted = cashed.get(pair[1], Sym_spell_func(pair[1]))\n",
        "        cashed[pair[0]] = predicted\n",
        "        if predicted == pair[0]:\n",
        "            correct += 1\n",
        "        total += 1\n",
        "        \n",
        "        if pair[0] == pair[1]:\n",
        "            total_correct += 1\n",
        "            if pair[0] !=  predicted:\n",
        "                correct_broken += 1\n",
        "        else:\n",
        "            total_mistaken += 1\n",
        "            if pair[0] == predicted:\n",
        "                mistaken_fixed += 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sClbim9-sAZN",
        "colab_type": "code",
        "outputId": "86a5b3f5-1ad7-4e35-d950-aaa6325baa97",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "print(correct/total)\n",
        "print(mistaken_fixed/total_mistaken)\n",
        "print(correct_broken/total_correct)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.7020979020979021\n",
            "0.5003837298541827\n",
            "0.26771563110141267\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w9KUOIFJthqa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def in_pro(input_word):\n",
        "  candidates_list = []\n",
        "  candidates_dict = {}\n",
        "  strongest_candidate = input_word\n",
        "  if input_word in unigrams:\n",
        "    return input_word\n",
        "  for i in deletes(input_word):\n",
        "    if i in glob_dict:\n",
        "      candidates_dict[unigrams[glob_dict[i]]] = glob_dict[i]\n",
        "      strongest_candidate = max(candidates_dict.keys())\n",
        "      strongest_candidate = candidates_dict[strongest_candidate]\n",
        "  return strongest_candidate"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yBetQrw1t1LA",
        "colab_type": "code",
        "outputId": "555137c0-8a47-4d5b-becd-2e3e5798aec5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "in_pro('надаело')"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'надоело'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "IQV37GPV3jvJ",
        "colab": {}
      },
      "source": [
        "correct = 0\n",
        "total = 0\n",
        "\n",
        "total_mistaken = 0\n",
        "mistaken_fixed = 0\n",
        "\n",
        "total_correct = 0\n",
        "correct_broken = 0\n",
        "\n",
        "cashed = {}\n",
        "for i in range(len(true)):\n",
        "    word_pairs = align_words(true[i], bad[i])\n",
        "    for pair in word_pairs:\n",
        "        predicted = cashed.get(pair[1], in_pro(pair[1]))\n",
        "        cashed[pair[0]] = predicted\n",
        "        if predicted == pair[0]:\n",
        "            correct += 1\n",
        "        total += 1\n",
        "        \n",
        "        if pair[0] == pair[1]:\n",
        "            total_correct += 1\n",
        "            if pair[0] !=  predicted:\n",
        "                correct_broken += 1\n",
        "        else:\n",
        "            total_mistaken += 1\n",
        "            if pair[0] == predicted:\n",
        "                mistaken_fixed += 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZgjWsCfBuI2M",
        "colab_type": "code",
        "outputId": "10b7c827-5480-4582-8fe6-19b5f2d2e9d8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "print(correct/total)\n",
        "print(mistaken_fixed/total_mistaken)\n",
        "print(correct_broken/total_correct)"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.6594405594405595\n",
            "0.36377590176515734\n",
            "0.2963133111289767\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qugzIznr2vhA",
        "colab_type": "text"
      },
      "source": [
        "### Триграммная модель"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nlkirrvGCXLd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "corpus_news = [['<start>', '<start>'] + sent + ['<end>'] for sent in corpus]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JweKqJNu27eC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "corpus = []\n",
        "for text in open('corpus_5000.txt').read().splitlines():\n",
        "    sents = sent_tokenize(text)\n",
        "    norm_sents = [normalize(sent) for sent in sents]\n",
        "    corpus += norm_sents"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iFLigV-421S4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def ngrammer(tokens, n=2):\n",
        "    ngrams = []\n",
        "    for i in range(0,len(tokens)-n+1):\n",
        "        ngrams.append(' '.join(tokens[i:i+n]))\n",
        "    return ngrams"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CVCxxzgM3JYP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "unigrams = Counter()\n",
        "bigrams = Counter()\n",
        "trigrams = Counter()\n",
        "\n",
        "for sentence in corpus_news:\n",
        "    unigrams.update(sentence)\n",
        "    bigrams.update(ngrammer(sentence))\n",
        "    trigrams.update(ngrammer(sentence, n=3))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zo3IgCDVCukx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def trigram_in_pro(input_words):\n",
        "  candidates_list = []\n",
        "  candidates_dict = {}\n",
        "  input_word = str(input_words.split()[-1])\n",
        "  strongest_candidate = input_word\n",
        "  if input_word in unigrams:\n",
        "    return input_word\n",
        "  for i in deletes(input_word):\n",
        "    if i in glob_dict:\n",
        "      one_word_prob = unigrams[glob_dict[i]]/len(unigrams)\n",
        "      bigram = \" \".join(input_words.split()[:-1]) \n",
        "      if unigrams[input_words.split()[0]]:\n",
        "        bigram_prob = bigrams[bigram]/unigrams[input_words.split()[0]]\n",
        "        trigram = bigram + \" \" + glob_dict[i]\n",
        "      if bigrams[bigram]:\n",
        "        trigram_prob = trigrams[trigram]/bigrams[bigram]\n",
        "        prob_with_context = one_word_prob*(1 + trigram_prob/bigram_prob)\n",
        "      else:\n",
        "         prob_with_context = one_word_prob\n",
        "      candidates_dict[unigrams[glob_dict[i]]] = glob_dict[i]\n",
        "      strongest_candidate = max(candidates_dict.keys())\n",
        "      strongest_candidate = candidates_dict[strongest_candidate]\n",
        "  return strongest_candidate"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G1m2PGR36Ks9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "3fe74f6b-b6ee-4a2b-a318-a1d51e8b65d9"
      },
      "source": [
        "trigram_in_pro(\"проходит со временим\")"
      ],
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'временами'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8dH3Z0P_6QDI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "correct = 0\n",
        "total = 0\n",
        "\n",
        "total_mistaken = 0\n",
        "mistaken_fixed = 0\n",
        "\n",
        "total_correct = 0\n",
        "correct_broken = 0\n",
        "\n",
        "cashed = {}\n",
        "for i in range(len(true)):\n",
        "  for j in range(len(true[i].split())):\n",
        "    word_pairs = align_words(true[i], bad[i])\n",
        "    if j == 0:\n",
        "      r_bigram = \"<start> <start> \"\n",
        "    elif j == len(true[i].split()):\n",
        "      r_bigram = true[i][j-1] + \" \" + true[i][j] + \" \"\n",
        "    else:\n",
        "      r_bigram = true[i][j-2] + \" \" + true[i][j-1] + \" \"\n",
        "    for pair in word_pairs:\n",
        "      predicted = cashed.get(pair[1], trigram_in_pro(r_bigram + pair[1]))\n",
        "      cashed[pair[0]] = predicted\n",
        "      if predicted == pair[0]:\n",
        "          correct += 1\n",
        "      total += 1\n",
        "      \n",
        "      if pair[0] == pair[1]:\n",
        "          total_correct += 1\n",
        "          if pair[0] !=  predicted:\n",
        "              correct_broken += 1\n",
        "      else:\n",
        "          total_mistaken += 1\n",
        "          if pair[0] == predicted:\n",
        "              mistaken_fixed += 1\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fSJKVyar7ipX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "07c0fe80-f63e-4837-a25a-0a94286ea148"
      },
      "source": [
        "print(correct/total)\n",
        "print(mistaken_fixed/total_mistaken)\n",
        "print(correct_broken/total_correct)"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.6662871064269258\n",
            "0.3399486633694359\n",
            "0.2951235943410893\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}