phonetics foreign word detector.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AAhmadS/NLP_HW2/blob/main/phonetics%20foreign%20word%20detector.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XKugniAeQcGt",
        "outputId": "4a861d93-7894-424f-85a3-78c95b404148"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "# mount Google Drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/drive/MyDrive/NLP"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GRCx-_IwWvxT",
        "outputId": "a72ce7b7-bf21-4ab1-cb26-e71bbbc39027"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/.shortcut-targets-by-id/1Ikn75GANDVFyb0klJe4-EBQJikt4_Ob_/NLP\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Extracting essential data\n",
        "\n"
      ],
      "metadata": {
        "id": "LYf9wqwEYwO4"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Essential packages"
      ],
      "metadata": {
        "id": "njh0oGO4Nr7G"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import locale\n",
        "def getpreferredencoding(do_setlocale = True):\n",
        "    return \"UTF-8\"\n",
        "locale.getpreferredencoding = getpreferredencoding"
      ],
      "metadata": {
        "id": "n2iCwQ7T3uBu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#installing the necessary libraries\n",
        "!pip install -q hazm\n",
        "!pip install -q dadmatools\n",
        "!pip install -q googletrans==3.1.0a0"
      ],
      "metadata": {
        "id": "znLvol8uYzV-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#importing the necessary libraries\n",
        "from __future__ import unicode_literals\n",
        "import hazm\n",
        "\n",
        "import dadmatools.pipeline.language as language\n",
        "\n",
        "import json\n",
        "import pickle\n",
        "import itertools\n",
        "from tqdm import tqdm\n",
        "import pandas as pd"
      ],
      "metadata": {
        "id": "vJ1FDzA_Y5VW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Creating dataset"
      ],
      "metadata": {
        "id": "QU-8QbkZN4Pw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"Nlp HW2/corpus.txt\",\"rb\") as f:\n",
        "  corpo = pickle.load(f)\n",
        "\n",
        "corpo"
      ],
      "metadata": {
        "id": "wtvAazXLhkjG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e57b4360-f694-4def-fe58-e7dc3fa3b5dd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['آئین',\n",
              " 'آب',\n",
              " 'آبا',\n",
              " 'آباجی',\n",
              " 'آباد',\n",
              " 'آبادسازی',\n",
              " 'آباده',\n",
              " 'آبادکننده',\n",
              " 'آبادگر',\n",
              " 'آبادی\\u200cنشین',\n",
              " 'آبان',\n",
              " 'آباژور',\n",
              " 'آباژورساز',\n",
              " 'آباژورسازی',\n",
              " 'آبتنی',\n",
              " 'آبتین',\n",
              " 'آبجو',\n",
              " 'آبجوسازی',\n",
              " 'آبجی',\n",
              " 'آبخازستان',\n",
              " 'آبخازیا',\n",
              " 'آبخوان',\n",
              " 'آبخور',\n",
              " 'آبخوری',\n",
              " 'آبخیز',\n",
              " 'آبخیزداری',\n",
              " 'آبخیزگاه',\n",
              " 'آبدار',\n",
              " 'آبدارخانه',\n",
              " 'آبدارچی',\n",
              " 'آبدزدک',\n",
              " 'آبدیده',\n",
              " 'آبدیدگی',\n",
              " 'آبراه',\n",
              " 'آبراهام',\n",
              " 'آبراهه',\n",
              " 'آبرفت',\n",
              " 'آبرنگ',\n",
              " 'آبرو',\n",
              " 'آبروبری',\n",
              " 'آبرودار',\n",
              " 'آبروداری',\n",
              " 'آبروریزی',\n",
              " 'آبرومند',\n",
              " 'آبرومندانه',\n",
              " 'آبریز',\n",
              " 'آبریزگاه',\n",
              " 'آبزی',\n",
              " 'آبزی\\u200cپروری',\n",
              " 'آبسال',\n",
              " 'آبستراکسیون',\n",
              " 'آبستره',\n",
              " 'آبستن',\n",
              " 'آبستنی',\n",
              " 'آبسه',\n",
              " 'آبسکون',\n",
              " 'آبشار',\n",
              " 'آبشامه',\n",
              " 'آبشخور',\n",
              " 'آبعلی',\n",
              " 'آبله',\n",
              " 'آبله\\u200cدار',\n",
              " 'آبله\\u200cرو',\n",
              " 'آبله\\u200cمرغان',\n",
              " 'آبله\\u200cکوبی',\n",
              " 'آبله\\u200cگون',\n",
              " 'آبنوس',\n",
              " 'آبونمان',\n",
              " 'آبپاش',\n",
              " 'آبکار',\n",
              " 'آبکاری',\n",
              " 'آبکش',\n",
              " 'آبکی',\n",
              " 'آبگرم',\n",
              " 'آبگوشت',\n",
              " 'آبگوشت\\u200cخور',\n",
              " 'آبگوشت\\u200cخوری',\n",
              " 'آبگون',\n",
              " 'آبگیر',\n",
              " 'آبگینه',\n",
              " 'آبیاری',\n",
              " 'آبی\\u200cپوش',\n",
              " 'آب\\u200cانبار',\n",
              " 'آب\\u200cاکسیژنه',\n",
              " 'آب\\u200cباریکه',\n",
              " 'آب\\u200cبازی',\n",
              " 'آب\\u200cبند',\n",
              " 'آب\\u200cبندی',\n",
              " 'آب\\u200cبها',\n",
              " 'آب\\u200cتنی',\n",
              " 'آب\\u200cجو',\n",
              " 'آب\\u200cجوفروش',\n",
              " 'آب\\u200cجوفروشی',\n",
              " 'آب\\u200cخوری',\n",
              " 'آب\\u200cداده',\n",
              " 'آب\\u200cدرمانی',\n",
              " 'آب\\u200cدزدک',\n",
              " 'آب\\u200cدیده',\n",
              " 'آب\\u200cرسان',\n",
              " 'آب\\u200cرسانی',\n",
              " 'آب\\u200cرفته',\n",
              " 'آب\\u200cروغن',\n",
              " 'آب\\u200cزدایی',\n",
              " 'آب\\u200cزده',\n",
              " 'آب\\u200cزن',\n",
              " 'آب\\u200cزیرکاه',\n",
              " 'آب\\u200cزیپو',\n",
              " 'آب\\u200cسردکن',\n",
              " 'آب\\u200cشده',\n",
              " 'آب\\u200cشناسی',\n",
              " 'آب\\u200cشنگولی',\n",
              " 'آب\\u200cشیرین\\u200cکن',\n",
              " 'آب\\u200cطلا',\n",
              " 'آب\\u200cغوره',\n",
              " 'آب\\u200cلمبو',\n",
              " 'آب\\u200cلنبو',\n",
              " 'آب\\u200cلیمو',\n",
              " 'آب\\u200cلیموخوری',\n",
              " 'آب\\u200cلیموگیری',\n",
              " 'آب\\u200cمروارید',\n",
              " 'آب\\u200cمیوه',\n",
              " 'آب\\u200cمیوه\\u200cفروش',\n",
              " 'آب\\u200cمیوه\\u200cفروشی',\n",
              " 'آب\\u200cمیوه\\u200cگیری',\n",
              " 'آب\\u200cنبات',\n",
              " 'آب\\u200cنما',\n",
              " 'آب\\u200cنمک',\n",
              " 'آب\\u200cوتاب',\n",
              " 'آب\\u200cپاش',\n",
              " 'آب\\u200cپاشی',\n",
              " 'آب\\u200cپز',\n",
              " 'آب\\u200cچکان',\n",
              " 'آب\\u200cژاول',\n",
              " 'آب\\u200cکردن',\n",
              " 'آب\\u200cکردنی',\n",
              " 'آب\\u200cکشی',\n",
              " 'آب\\u200cکشیده',\n",
              " 'آب\\u200cگرفته',\n",
              " 'آب\\u200cگرفتگی',\n",
              " 'آب\\u200cگرم\\u200cکن',\n",
              " 'آب\\u200cگوشت',\n",
              " 'آب\\u200cگیری',\n",
              " 'آتاتورک',\n",
              " 'آتاری',\n",
              " 'آتش',\n",
              " 'آتشبار',\n",
              " 'آتشخانه',\n",
              " 'آتشدان',\n",
              " 'آتشزنه',\n",
              " 'آتشکده',\n",
              " 'آتشگاه',\n",
              " 'آتشین',\n",
              " 'آتشین\\u200cمزاج',\n",
              " 'آتش\\u200cافروز',\n",
              " 'آتش\\u200cافروزی',\n",
              " 'آتش\\u200cبار',\n",
              " 'آتش\\u200cباران',\n",
              " 'آتش\\u200cبازی',\n",
              " 'آتش\\u200cبس',\n",
              " 'آتش\\u200cبیار',\n",
              " 'آتش\\u200cخانه',\n",
              " 'آتش\\u200cزا',\n",
              " 'آتش\\u200cسوزی',\n",
              " 'آتش\\u200cفشان',\n",
              " 'آتش\\u200cفشانی',\n",
              " 'آتش\\u200cنشان',\n",
              " 'آتش\\u200cنشانی',\n",
              " 'آتش\\u200cپاره',\n",
              " 'آتش\\u200cپرست',\n",
              " 'آتش\\u200cپرستی',\n",
              " 'آتش\\u200cگردان',\n",
              " 'آتش\\u200cگرفته',\n",
              " 'آتش\\u200cگیر',\n",
              " 'آتل',\n",
              " 'آتلانتا',\n",
              " 'آتلانتیس',\n",
              " 'آتلانتیک',\n",
              " 'آتلیه',\n",
              " 'آتل\\u200cبندی',\n",
              " 'آتل\\u200cپیچ',\n",
              " 'آتن',\n",
              " 'آتنا',\n",
              " 'آتنه',\n",
              " 'آتوسا',\n",
              " 'آتی',\n",
              " 'آتیلا',\n",
              " 'آتیه',\n",
              " 'آتیه\\u200cدار',\n",
              " 'آتی\\u200cساز',\n",
              " 'آت\\u200cآشغال',\n",
              " 'آثار',\n",
              " 'آثارالباقیه',\n",
              " 'آثاری',\n",
              " 'آج',\n",
              " 'آجان',\n",
              " 'آجر',\n",
              " 'آجرنما',\n",
              " 'آجرپاره',\n",
              " 'آجرپز',\n",
              " 'آجرپزی',\n",
              " 'آجرچینی',\n",
              " 'آجرکاری',\n",
              " 'آجل',\n",
              " 'آجودان',\n",
              " 'آجودانیه',\n",
              " 'آجی',\n",
              " 'آجیده',\n",
              " 'آجیل',\n",
              " 'آجیل\\u200cخوری',\n",
              " 'آجیل\\u200cفروش',\n",
              " 'آجیل\\u200cفروشی',\n",
              " 'آحاد',\n",
              " 'آخ',\n",
              " 'آخته',\n",
              " 'آخته',\n",
              " 'آخر',\n",
              " 'آخرالامر',\n",
              " 'آخرالزمان',\n",
              " 'آخرت',\n",
              " 'آخرت\\u200cگرا',\n",
              " 'آخرین',\n",
              " 'آخور',\n",
              " 'آخوند',\n",
              " 'آخوندزاده',\n",
              " 'آخ\\u200cواوخ',\n",
              " 'آداب',\n",
              " 'آداب\\u200cالصلوه',\n",
              " 'آداب\\u200cدان',\n",
              " 'آدام',\n",
              " 'آدامز',\n",
              " 'آدامس',\n",
              " 'آدامس\\u200cفروش',\n",
              " 'آدامس\\u200cفروشی',\n",
              " 'آدرس\\u200cدهی',\n",
              " 'آدرس\\u200cنویسی',\n",
              " 'آدرنال',\n",
              " 'آدرنالین',\n",
              " 'آدریان',\n",
              " 'آدلاید',\n",
              " 'آدلر',\n",
              " 'آدلف',\n",
              " 'آدم',\n",
              " 'آدمک',\n",
              " 'آدمکش',\n",
              " 'آدمیت',\n",
              " 'آدمیزاد',\n",
              " 'آدم\\u200cآهنی',\n",
              " 'آدم\\u200cبرفی',\n",
              " 'آدم\\u200cخوار',\n",
              " 'آدم\\u200cخواری',\n",
              " 'آدم\\u200cربا',\n",
              " 'آدم\\u200cربایی',\n",
              " 'آدم\\u200cشناس',\n",
              " 'آدم\\u200cفروش',\n",
              " 'آدم\\u200cفروشی',\n",
              " 'آدم\\u200cندیده',\n",
              " 'آدم\\u200cنما',\n",
              " 'آدولف',\n",
              " 'آدیداس',\n",
              " 'آدیس\\u200cآبابا',\n",
              " 'آدینه',\n",
              " 'آذر',\n",
              " 'آذربایجان',\n",
              " 'آذرخش',\n",
              " 'آذرشهر',\n",
              " 'آذرماه',\n",
              " 'آذرنژاد',\n",
              " 'آذرین',\n",
              " 'آذری\\u200cتبار',\n",
              " 'آذری\\u200cزبان',\n",
              " 'آذوقه',\n",
              " 'آذین',\n",
              " 'آذین\\u200cبخش',\n",
              " 'آذین\\u200cشده',\n",
              " 'آرا',\n",
              " 'آراء',\n",
              " 'آرارات',\n",
              " 'آراستن',\n",
              " 'آراسته',\n",
              " 'آراسته',\n",
              " 'آراسته\\u200cشده',\n",
              " 'آراستگی',\n",
              " 'آرام',\n",
              " 'آرامش',\n",
              " 'آرامش\\u200cبخش',\n",
              " 'آرامش\\u200cدهنده',\n",
              " 'آرامگاه',\n",
              " 'آرامی',\n",
              " 'آرامیده',\n",
              " 'آرام\\u200cآرام',\n",
              " 'آرام\\u200cبخش',\n",
              " 'آرام\\u200cسازی',\n",
              " 'آرام\\u200cپز',\n",
              " 'آرام\\u200cکننده',\n",
              " 'آرام\\u200cگرفته',\n",
              " 'آران',\n",
              " 'آراگون',\n",
              " 'آرایش',\n",
              " 'آرایشگاه',\n",
              " 'آرایشگر',\n",
              " 'آرایش\\u200cشده',\n",
              " 'آرایش\\u200cکرده',\n",
              " 'آراینده',\n",
              " 'آرایه',\n",
              " 'آرایه\\u200cشناس',\n",
              " 'آرتروز',\n",
              " 'آرتریت',\n",
              " 'آرتمیس',\n",
              " 'آرتور',\n",
              " 'آرتیست',\n",
              " 'آرد',\n",
              " 'آردسازی',\n",
              " 'آردفروشی',\n",
              " 'آرزو',\n",
              " 'آرزومند',\n",
              " 'آرزومندانه',\n",
              " 'آرزوکننده',\n",
              " 'آرسن',\n",
              " 'آرسنال',\n",
              " 'آرسنیک',\n",
              " 'آرش',\n",
              " 'آرشیو',\n",
              " 'آرشیوبندی',\n",
              " 'آرشیوشده',\n",
              " 'آرم',\n",
              " 'آرماتور',\n",
              " 'آرماتوربند',\n",
              " 'آرمان',\n",
              " 'آرمان\\u200cخواه',\n",
              " 'آرمان\\u200cخواهانه',\n",
              " 'آرمان\\u200cخواهی',\n",
              " 'آرمان\\u200cشهر',\n",
              " 'آرمان\\u200cگرا',\n",
              " 'آرمان\\u200cگرایانه',\n",
              " 'آرمان\\u200cگرایی',\n",
              " 'آرمسترانگ',\n",
              " 'آرمیدن',\n",
              " 'آرمیده',\n",
              " 'آرمیدگی',\n",
              " 'آرمین',\n",
              " 'آرمیچر',\n",
              " 'آرنج',\n",
              " 'آرنولد',\n",
              " 'آرواره',\n",
              " 'آرواره\\u200cدار',\n",
              " 'آروغ',\n",
              " 'آروماتیک',\n",
              " 'آرپی\\u200cجی',\n",
              " 'آرژانتین',\n",
              " 'آرکانزاس',\n",
              " 'آرگومان',\n",
              " 'آرگون',\n",
              " 'آری',\n",
              " 'آریا',\n",
              " 'آریامهر',\n",
              " 'آریانا',\n",
              " 'آریانپور',\n",
              " 'آریایی',\n",
              " 'آریزونا',\n",
              " 'آریل',\n",
              " 'آرین',\n",
              " 'آریوبرزن',\n",
              " 'آز',\n",
              " 'آزاد',\n",
              " 'آزاداندیش',\n",
              " 'آزاداندیشانه',\n",
              " 'آزاداندیشی',\n",
              " 'آزادانه',\n",
              " 'آزادراه',\n",
              " 'آزادسازی',\n",
              " 'آزادشده',\n",
              " 'آزادشهر',\n",
              " 'آزادمرد',\n",
              " 'آزادمنش',\n",
              " 'آزادمنشانه',\n",
              " 'آزاده',\n",
              " 'آزاده\\u200cپرور',\n",
              " 'آزادکار',\n",
              " 'آزادکرده',\n",
              " 'آزادکننده',\n",
              " 'آزادگان',\n",
              " 'آزادگی',\n",
              " 'آزادیخواه',\n",
              " 'آزادی\\u200cبخش',\n",
              " 'آزادی\\u200cجویانه',\n",
              " 'آزادی\\u200cخواه',\n",
              " 'آزادی\\u200cخواهانه',\n",
              " 'آزادی\\u200cخواهی',\n",
              " 'آزادی\\u200cستیز',\n",
              " 'آزادی\\u200cطلب',\n",
              " 'آزادی\\u200cطلبی',\n",
              " 'آزار',\n",
              " 'آزاردهنده',\n",
              " 'آزاردهندگی',\n",
              " 'آزاردیده',\n",
              " 'آزارنده',\n",
              " 'آزبست',\n",
              " 'آزردن',\n",
              " 'آزرده',\n",
              " 'آزرده',\n",
              " 'آزرده\\u200cخاطر',\n",
              " 'آزرده\\u200cدل',\n",
              " 'آزردگی',\n",
              " 'آزرم',\n",
              " 'آزمایش',\n",
              " 'آزمایشگاه',\n",
              " 'آزمایشگر',\n",
              " 'آزمایش\\u200cشده',\n",
              " 'آزمایش\\u200cشونده',\n",
              " 'آزمایش\\u200cکننده',\n",
              " 'آزماینده',\n",
              " 'آزمند',\n",
              " 'آزمودن',\n",
              " 'آزموده',\n",
              " 'آزمودگی',\n",
              " 'آزمون',\n",
              " 'آزمون\\u200cپذیر',\n",
              " 'آزگار',\n",
              " 'آزیتا',\n",
              " 'آسان',\n",
              " 'آسانسور',\n",
              " 'آسانسورسازی',\n",
              " 'آسان\\u200cسازی',\n",
              " 'آسان\\u200cفهم',\n",
              " 'آسان\\u200cگیری',\n",
              " 'آسایش',\n",
              " 'آسایشگاه',\n",
              " 'آسایش\\u200cطلب',\n",
              " 'آسایش\\u200cطلبی',\n",
              " 'آستارا',\n",
              " 'آستان',\n",
              " 'آستانه',\n",
              " 'آستانه\\u200cاشرفیه',\n",
              " 'آستر',\n",
              " 'آستین',\n",
              " 'آستیگمات',\n",
              " 'آسفالت',\n",
              " 'آسفالته',\n",
              " 'آسفالت\\u200cریزی',\n",
              " 'آسفالت\\u200cکار',\n",
              " 'آسفالت\\u200cکاری',\n",
              " 'آسم',\n",
              " 'آسمان',\n",
              " 'آسمان\\u200cآباد',\n",
              " 'آسمان\\u200cجل',\n",
              " 'آسمان\\u200cخراش',\n",
              " 'آسمان\\u200cغرنبه',\n",
              " 'آسمان\\u200cنما',\n",
              " 'آسودن',\n",
              " 'آسوده',\n",
              " 'آسوده',\n",
              " 'آسوده\\u200cخاطر',\n",
              " 'آسوده\\u200cخیال',\n",
              " 'آسوده\\u200cخیالی',\n",
              " 'آسوده\\u200cدل',\n",
              " 'آسودگی',\n",
              " 'آسوشیتدپرس',\n",
              " 'آسپرین',\n",
              " 'آسپیرین',\n",
              " 'آسکاریس',\n",
              " 'آسیا',\n",
              " 'آسیاب',\n",
              " 'آسیایی',\n",
              " 'آسیب',\n",
              " 'آسیب\\u200cدیده',\n",
              " 'آسیب\\u200cدیدگی',\n",
              " 'آسیب\\u200cرسان',\n",
              " 'آسیب\\u200cرسانی',\n",
              " 'آسیب\\u200cزا',\n",
              " 'آسیب\\u200cزده',\n",
              " 'آسیب\\u200cزدگی',\n",
              " 'آسیب\\u200cزننده',\n",
              " 'آسیب\\u200cشناس',\n",
              " 'آسیب\\u200cشناسی',\n",
              " 'آسیب\\u200cناپذیر',\n",
              " 'آسیب\\u200cندیده',\n",
              " 'آسیب\\u200cپذیر',\n",
              " 'آسیب\\u200cپذیری',\n",
              " 'آسیمه',\n",
              " 'آسیمه\\u200cسر',\n",
              " 'آسیموف',\n",
              " 'آسیه',\n",
              " 'آس\\u200cوپاس',\n",
              " 'آش',\n",
              " 'آشامانده',\n",
              " 'آشامانیده',\n",
              " 'آشامیدن',\n",
              " 'آشامیده',\n",
              " 'آشامیده',\n",
              " 'آشتی',\n",
              " 'آشتیان',\n",
              " 'آشتی\\u200cجویانه',\n",
              " 'آشتی\\u200cجویی',\n",
              " 'آشتی\\u200cدهنده',\n",
              " 'آشتی\\u200cطلبانه',\n",
              " 'آشتی\\u200cناپذیر',\n",
              " 'آشتی\\u200cپذیر',\n",
              " 'آشتی\\u200cکن',\n",
              " 'آشتی\\u200cکننده',\n",
              " 'آشغال',\n",
              " 'آشغالدان',\n",
              " 'آشغال\\u200cجمع\\u200cکن',\n",
              " 'آشغال\\u200cدان',\n",
              " 'آشغال\\u200cفروش',\n",
              " 'آشفتن',\n",
              " 'آشفته',\n",
              " 'آشفته',\n",
              " 'آشفته\\u200cبازار',\n",
              " 'آشفته\\u200cحال',\n",
              " 'آشفته\\u200cخاطر',\n",
              " 'آشفتگی',\n",
              " 'آشنا',\n",
              " 'آشنازدایی',\n",
              " 'آشناساز',\n",
              " 'آشنایی',\n",
              " 'آشنایی\\u200cزدایی',\n",
              " 'آشوب',\n",
              " 'آشوبگر',\n",
              " 'آشوبگرانه',\n",
              " 'آشوبیده',\n",
              " 'آشوب\\u200cآفرینی',\n",
              " 'آشوب\\u200cزده',\n",
              " 'آشوب\\u200cطلب',\n",
              " 'آشوب\\u200cطلبانه',\n",
              " 'آشور',\n",
              " 'آشپز',\n",
              " 'آشپزباشی',\n",
              " 'آشپزخانه',\n",
              " 'آشکار',\n",
              " 'آشکارا',\n",
              " 'آشکارساز',\n",
              " 'آشکارشده',\n",
              " 'آشکارکننده',\n",
              " 'آشیان',\n",
              " 'آشیانه',\n",
              " 'آشیل',\n",
              " 'آش\\u200cخور',\n",
              " 'آصف',\n",
              " 'آغاز',\n",
              " 'آغازشده',\n",
              " 'آغازکننده',\n",
              " 'آغازگر',\n",
              " 'آغازی',\n",
              " 'آغازیده',\n",
              " 'آغازین',\n",
              " 'آغاسی',\n",
              " 'آغالیده',\n",
              " 'آغداشلو',\n",
              " 'آغشتن',\n",
              " 'آغشتنی',\n",
              " 'آغشته',\n",
              " 'آغشته',\n",
              " 'آغشتگی',\n",
              " 'آغل',\n",
              " 'آغوز',\n",
              " 'آغوش',\n",
              " 'آفات',\n",
              " 'آفاق',\n",
              " 'آفاقی',\n",
              " 'آفت',\n",
              " 'آفتاب',\n",
              " 'آفتابه',\n",
              " 'آفتابگردان',\n",
              " 'آفتاب\\u200cخورده',\n",
              " 'آفتاب\\u200cزده',\n",
              " 'آفتاب\\u200cزدگی',\n",
              " 'آفتاب\\u200cسوخته',\n",
              " 'آفتاب\\u200cسوختگی',\n",
              " 'آفتاب\\u200cنزده',\n",
              " 'آفتاب\\u200cپرست',\n",
              " 'آفتاب\\u200cپرستی',\n",
              " 'آفتاب\\u200cگردان',\n",
              " 'آفتاب\\u200cگیر',\n",
              " 'آفت\\u200cدیده',\n",
              " 'آفت\\u200cرسیده',\n",
              " 'آفت\\u200cزا',\n",
              " 'آفت\\u200cزده',\n",
              " 'آفت\\u200cزدگی',\n",
              " 'آفت\\u200cپذیر',\n",
              " 'آفت\\u200cکش',\n",
              " 'آفریدن',\n",
              " 'آفریدنی',\n",
              " 'آفریده',\n",
              " 'آفریده',\n",
              " 'آفریده\\u200cشده',\n",
              " 'آفریدگار',\n",
              " 'آفریدگان',\n",
              " 'آفریقا',\n",
              " 'آفریقایی',\n",
              " 'آفریقایی\\u200cتبار',\n",
              " 'آفرین',\n",
              " 'آفرینش',\n",
              " 'آفریننده',\n",
              " 'آفرینندگی',\n",
              " 'آفرین\\u200cگو',\n",
              " 'آفساید',\n",
              " 'آفسایدگیری',\n",
              " 'آفست',\n",
              " 'آق',\n",
              " 'آقا',\n",
              " 'آقابالاسر',\n",
              " 'آقابزرگ',\n",
              " 'آقاجان',\n",
              " 'آقاجری',\n",
              " 'آقاخان',\n",
              " 'آقاخانی',\n",
              " 'آقازاده',\n",
              " 'آقاسی',\n",
              " 'آقامنش',\n",
              " 'آقامنشانه',\n",
              " 'آق\\u200cبانو',\n",
              " 'آق\\u200cقلا',\n",
              " 'آل',\n",
              " 'آلاء',\n",
              " 'آلاباما',\n",
              " 'آلات',\n",
              " 'آلاسکا',\n",
              " 'آلاله',\n",
              " 'آلام',\n",
              " 'آلاچیق',\n",
              " 'آلاگارسون',\n",
              " 'آلایش',\n",
              " 'آلاینده',\n",
              " 'آلایندگی',\n",
              " 'آلبالو',\n",
              " 'آلبالوخشکه',\n",
              " 'آلبالوپلو',\n",
              " 'آلبالویی\\u200cرنگ',\n",
              " 'آلبانی',\n",
              " 'آلبانیایی',\n",
              " 'آلبانیایی\\u200cتبار',\n",
              " 'آلبانی\\u200cزبان',\n",
              " 'آلبر',\n",
              " 'آلبرت',\n",
              " 'آلبرتا',\n",
              " 'آلبرتو',\n",
              " 'آلبرکامو',\n",
              " 'آلبومین',\n",
              " 'آلت',\n",
              " 'آلخاندرو',\n",
              " 'آلرژی',\n",
              " 'آلرژیک',\n",
              " 'آلرژی\\u200cزا',\n",
              " 'آلزایمر',\n",
              " 'آلفا',\n",
              " 'آلفرد',\n",
              " 'آلما',\n",
              " 'آلمان',\n",
              " 'آلمانی\\u200cزبان',\n",
              " 'آلن',\n",
              " 'آلنده',\n",
              " 'آلن\\u200cپو',\n",
              " 'آلو',\n",
              " 'آلودن',\n",
              " 'آلوده',\n",
              " 'آلوده',\n",
              " 'آلوده\\u200cدامان',\n",
              " 'آلوده\\u200cساز',\n",
              " 'آلوده\\u200cشده',\n",
              " 'آلوده\\u200cکننده',\n",
              " 'آلودگی',\n",
              " 'آلودگی\\u200cزدایی',\n",
              " 'آلوزرد',\n",
              " 'آلوفروش',\n",
              " 'آلوفروشی',\n",
              " 'آلومین',\n",
              " 'آلومینیوم',\n",
              " 'آلومینیوم\\u200cدار',\n",
              " 'آلومینیوم\\u200cسازی',\n",
              " 'آلونک',\n",
              " 'آلوچه',\n",
              " 'آلپ',\n",
              " 'آلپاین',\n",
              " 'آلی',\n",
              " 'آلیاژ',\n",
              " 'آلیس',\n",
              " 'آل\\u200cبویه',\n",
              " 'آل\\u200cزده',\n",
              " 'آل\\u200cزدگی',\n",
              " 'آل\\u200cعمران',\n",
              " 'آل\\u200cپاچینو',\n",
              " 'آماج',\n",
              " 'آماجگاه',\n",
              " 'آماد',\n",
              " 'آماده',\n",
              " 'آماده\\u200cباش',\n",
              " 'آماده\\u200cساز',\n",
              " 'آماده\\u200cشده',\n",
              " 'آمادگاه',\n",
              " 'آمادگی',\n",
              " 'آمار',\n",
              " 'آمارنامه',\n",
              " 'آماره',\n",
              " 'آمارگرفته',\n",
              " 'آمارگونه',\n",
              " 'آمارگیر',\n",
              " 'آمارگیرنده',\n",
              " 'آمارگیری',\n",
              " 'آمازون',\n",
              " 'آماس',\n",
              " 'آماسانده',\n",
              " 'آماسانیده',\n",
              " 'آماسیدن',\n",
              " 'آمال',\n",
              " 'آمایش',\n",
              " 'آمبولانس',\n",
              " 'آمدن',\n",
              " 'آمدنی',\n",
              " 'آمده',\n",
              " 'آمر',\n",
              " 'آمرانه',\n",
              " 'آمرزش',\n",
              " 'آمرزش\\u200cخواه',\n",
              " 'آمرزنده',\n",
              " 'آمرزگار',\n",
              " 'آمرزیدن',\n",
              " 'آمرزیدنی',\n",
              " 'آمرزیده',\n",
              " 'آمرزیده',\n",
              " 'آمرزیدگی',\n",
              " 'آمرین',\n",
              " 'آمریکا',\n",
              " 'آمریکائی',\n",
              " 'آمریکازدگی',\n",
              " 'آمریکایی',\n",
              " 'آمریکایی\\u200cتبار',\n",
              " 'آمستردام',\n",
              " 'آمفی\\u200cتئاتر',\n",
              " 'آمل',\n",
              " 'آمنه',\n",
              " 'آموختن',\n",
              " 'آموختنی',\n",
              " 'آموخته',\n",
              " 'آموخته',\n",
              " 'آموخته\\u200cشده',\n",
              " 'آمودریا',\n",
              " 'آمور',\n",
              " 'آموزانده',\n",
              " 'آموزش',\n",
              " 'آموزشکده',\n",
              " 'آموزشگاه',\n",
              " 'آموزشگر',\n",
              " 'آموزشیار',\n",
              " 'آموزش\\u200cدهنده',\n",
              " 'آموزش\\u200cدیده',\n",
              " 'آموزش\\u200cگیرنده',\n",
              " 'آموزش\\u200cیافته',\n",
              " 'آموزنده',\n",
              " 'آموزه',\n",
              " 'آموزگار',\n",
              " 'آمون',\n",
              " 'آمونیاک',\n",
              " 'آمونیوم',\n",
              " 'آمپر',\n",
              " 'آمپرساعت',\n",
              " 'آمپرسنج',\n",
              " 'آمپرمتر',\n",
              " 'آمپلی\\u200cفایر',\n",
              " 'آمپول',\n",
              " 'آمپی\\u200cسیلین',\n",
              " 'آمیب',\n",
              " 'آمیختن',\n",
              " 'آمیختنی',\n",
              " 'آمیخته',\n",
              " 'آمیخته',\n",
              " 'آمیخته\\u200cشده',\n",
              " 'آمیختگی',\n",
              " 'آمیزش',\n",
              " 'آمیزنده',\n",
              " 'آمیزه',\n",
              " 'آمین',\n",
              " 'آمینه',\n",
              " 'آن',\n",
              " 'آنا',\n",
              " 'آنات',\n",
              " 'آناتولی',\n",
              " 'آناتومیک',\n",
              " 'آنالوگ',\n",
              " 'آنالیز',\n",
              " 'آنالیزور',\n",
              " 'آناناس',\n",
              " 'آناهیتا',\n",
              " 'آناکوندا',\n",
              " 'آنتالیا',\n",
              " 'آنتروپی',\n",
              " 'آنتن',\n",
              " 'آنتوان',\n",
              " 'آنتون',\n",
              " 'آنتونوف',\n",
              " 'آنتونی',\n",
              " 'آنتونیو',\n",
              " 'آنتیل',\n",
              " 'آنتیک\\u200cفروش',\n",
              " 'آنتیک\\u200cفروشی',\n",
              " 'آنتی\\u200cاسید',\n",
              " 'آنتی\\u200cبیوتیک\\u200cدار',\n",
              " 'آنتی\\u200cتوکسین',\n",
              " 'آنجا',\n",
              " 'آنجل',\n",
              " 'آنجلینا',\n",
              " 'آند',\n",
              " 'آندرانیک',\n",
              " 'آندره',\n",
              " 'آندریاس',\n",
              " 'آندلس',\n",
              " 'آندورا',\n",
              " 'آندومتر',\n",
              " 'آندی',\n",
              " 'آنزیم\\u200cساز',\n",
              " 'آنفلوانزا',\n",
              " 'آنفولانزا',\n",
              " 'آنلاین',\n",
              " 'آنوفل',\n",
              " 'آنومالی',\n",
              " 'آنچه',\n",
              " 'آنک',\n",
              " 'آنکارا',\n",
              " 'آنکه',\n",
              " 'آنگاه',\n",
              " 'آنگستروم',\n",
              " 'آنگلوساکسون',\n",
              " 'آنگولا',\n",
              " 'آنگولایی',\n",
              " 'آنی',\n",
              " 'آنیون',\n",
              " 'آن\\u200cسو',\n",
              " 'آن\\u200cطرف',\n",
              " 'آن\\u200cطرف\\u200cتر',\n",
              " 'آن\\u200cقدر',\n",
              " 'آن\\u200cچنان',\n",
              " 'آن\\u200cکه',\n",
              " 'آه',\n",
              " 'آها',\n",
              " 'آهار',\n",
              " 'آهاردار',\n",
              " 'آهارزده',\n",
              " 'آهان',\n",
              " 'آهای',\n",
              " 'آهسته',\n",
              " 'آهسته\\u200cآهسته',\n",
              " 'آهستگی',\n",
              " 'آهن',\n",
              " 'آهنجیده',\n",
              " 'آهنربا',\n",
              " 'آهنگ',\n",
              " 'آهنگر',\n",
              " 'آهنگری',\n",
              " 'آهنگساز',\n",
              " 'آهنگین',\n",
              " 'آهنگ\\u200cساز',\n",
              " 'آهنگ\\u200cسازی',\n",
              " 'آهنی',\n",
              " 'آهنین',\n",
              " 'آهنین\\u200cپنجه',\n",
              " 'آهن\\u200cآلات',\n",
              " 'آهن\\u200cبر',\n",
              " 'آهن\\u200cبری',\n",
              " 'آهن\\u200cربا',\n",
              " 'آهن\\u200cربایی',\n",
              " 'آهن\\u200cفروش',\n",
              " 'آهن\\u200cفروشی',\n",
              " 'آهن\\u200cقراضه',\n",
              " 'آهن\\u200cپاره',\n",
              " 'آهن\\u200cکار',\n",
              " 'آهو',\n",
              " 'آهوان',\n",
              " 'آهک',\n",
              " 'آهکی',\n",
              " 'آهک\\u200cدار',\n",
              " 'آهیختن',\n",
              " 'آهیخته',\n",
              " 'آوا',\n",
              " 'آوار',\n",
              " 'آواربرداری',\n",
              " 'آواره',\n",
              " 'آوارگی',\n",
              " 'آواز',\n",
              " 'آوازخوان',\n",
              " 'آوازخوانی',\n",
              " 'آوازه',\n",
              " 'آوازه\\u200cخوان',\n",
              " 'آوازه\\u200cخوانی',\n",
              " 'آواشناختی',\n",
              " 'آواشناس',\n",
              " 'آواشناسی',\n",
              " 'آوانتاژ',\n",
              " 'آوانس',\n",
              " 'آوانویسی',\n",
              " 'آوانگار',\n",
              " 'آوانگارد',\n",
              " 'آوانگاری',\n",
              " 'آوردن',\n",
              " 'آوردنی',\n",
              " 'آورده',\n",
              " 'آوردگاه',\n",
              " 'آورنده',\n",
              " 'آوریل',\n",
              " 'آوند',\n",
              " 'آونگ',\n",
              " 'آووگادرو',\n",
              " 'آویختن',\n",
              " 'آویختنی',\n",
              " 'آویخته',\n",
              " 'آویخته',\n",
              " 'آویختگی',\n",
              " 'آویز',\n",
              " 'آویزان',\n",
              " 'آویزه',\n",
              " 'آویشن',\n",
              " 'آویشن\\u200cدار',\n",
              " 'آپادانا',\n",
              " 'آپارات',\n",
              " 'آپاراتچی',\n",
              " 'آپاراتی',\n",
              " 'آپارات\\u200cخانه',\n",
              " 'آپارتاید',\n",
              " 'آپارتمان\\u200cساز',\n",
              " 'آپارتمان\\u200cسازی',\n",
              " 'آپارتمان\\u200cنشین',\n",
              " 'آپارتمان\\u200cنشینی',\n",
              " 'آپاندیس',\n",
              " 'آپاچی',\n",
              " 'آپلود',\n",
              " 'آپولو',\n",
              " 'آچار',\n",
              " 'آچارفرانسه',\n",
              " 'آژانس',\n",
              " 'آژاکس',\n",
              " 'آژیر',\n",
              " 'آژیرکش',\n",
              " 'آکاردئون',\n",
              " 'آکبند',\n",
              " 'آکروبات',\n",
              " 'آکروباتیک',\n",
              " 'آکروپولیس',\n",
              " 'آکریلیک',\n",
              " 'آکسان',\n",
              " 'آکسفورد',\n",
              " 'آکله',\n",
              " 'آکندن',\n",
              " 'آکنده',\n",
              " 'آکنده',\n",
              " 'آکندگی',\n",
              " 'آکنه',\n",
              " 'آکورد',\n",
              " 'آکوستیک',\n",
              " 'آگاتا',\n",
              " 'آگامنون',\n",
              " 'آگاه',\n",
              " 'آگاهاندن',\n",
              " 'آگاهانه',\n",
              " 'آگاهی',\n",
              " 'آگاهی\\u200cبخش',\n",
              " 'آگاهی\\u200cدهنده',\n",
              " 'آگاه\\u200cسازنده',\n",
              " 'آگاه\\u200cسازی',\n",
              " 'آگاه\\u200cکننده',\n",
              " 'آگه',\n",
              " 'آگهی',\n",
              " 'آگهی\\u200cدهنده',\n",
              " 'آگهی\\u200cدهندگان',\n",
              " 'آیا',\n",
              " 'آیات',\n",
              " 'آیت',\n",
              " 'آیتم',\n",
              " 'آیتم\\u200cسازی',\n",
              " 'آیت\\u200cالله',\n",
              " 'آیت\\u200cالکرسی',\n",
              " 'آیدا',\n",
              " 'آیدین',\n",
              " 'آیزاک',\n",
              " 'آیزنهاور',\n",
              " 'آیفون',\n",
              " 'آیلتس',\n",
              " 'آینده',\n",
              " 'آینده\\u200cبینی',\n",
              " 'آینده\\u200cساز',\n",
              " 'آینده\\u200cنگر',\n",
              " 'آینده\\u200cنگرانه',\n",
              " 'آینده\\u200cنگری',\n",
              " 'آینده\\u200cپژوه',\n",
              " 'آیندگان',\n",
              " 'آینه',\n",
              " 'آینه\\u200cبندان',\n",
              " 'آینه\\u200cبندی',\n",
              " 'آینه\\u200cکار',\n",
              " 'آینه\\u200cکاری',\n",
              " 'آیه',\n",
              " 'آیووا',\n",
              " 'آیکون',\n",
              " 'آیین',\n",
              " 'آیینه',\n",
              " 'آیینه\\u200cبندان',\n",
              " 'آیینه\\u200cخانه',\n",
              " 'آیینه\\u200cکاری',\n",
              " 'آیینی',\n",
              " 'آیین\\u200cبندی',\n",
              " 'آیین\\u200cمند',\n",
              " 'آیین\\u200cنامه',\n",
              " 'آیین\\u200cپرست',\n",
              " ...]"
            ]
          },
          "metadata": {},
          "execution_count": 184
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#the corpus used for this task comes from hazm word list\n",
        "corpus = list(set(corpo))\n",
        "len(corpus)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kLtriJNNo35v",
        "outputId": "9d6a0b27-04f3-40ab-b912-2abbc1423ef0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "35316"
            ]
          },
          "metadata": {},
          "execution_count": 185
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# !pip install sentencepiece\n",
        "# !pip install transformers\n",
        "# from transformers import AutoTokenizer\n",
        "# from transformers import AutoModelForTokenClassification  # for pytorch\n",
        "# from transformers import TFAutoModelForTokenClassification  # for tensorflow\n",
        "# from transformers import pipeline\n",
        "# # model_name_or_path = \"HooshvareLab/bert-fa-zwnj-base-ner\"  # Roberta\n",
        "# # model_name_or_path = \"HooshvareLab/roberta-fa-zwnj-base-ner\"  # Roberta\n",
        "# model_name_or_path = \"HooshvareLab/distilbert-fa-zwnj-base-ner\"  # Distilbert\n",
        "# # model_name_or_path = \"HooshvareLab/albert-fa-zwnj-base-v2-ner\"  # Albert\n",
        "\n",
        "# tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)\n",
        "\n",
        "# # tokenizer=hazm.WordTokenizer\n",
        "# model = AutoModelForTokenClassification.from_pretrained(model_name_or_path)  # Pytorch\n",
        "# # model = TFAutoModelForTokenClassification.from_pretrained(model_name_or_path)  # Tensorflow\n",
        "\n",
        "# ner = pipeline(\"ner\", model=model, tokenizer=tokenizer)\n"
      ],
      "metadata": {
        "id": "QxGwIUZP6aMO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(corpus)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-AN8OwAC7ANC",
        "outputId": "e62aef6c-733a-44d1-a4cc-fe32e0b6a493"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "35316"
            ]
          },
          "metadata": {},
          "execution_count": 187
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# persian_names=pd.read_csv(\"Nlp HW2/Persian names.csv\")\n",
        "# persian_names_list=persian_names[\"first_name\"]\n",
        "# persian_names_list=persian_names_list.tolist()\n",
        "# for word in corpus:\n",
        "#   ner_res=ner(word)\n",
        "#   if len(ner_res)==0:\n",
        "#     continue\n",
        "#   if ner(word)[0][\"entity\"]==\"B-PER\" and word not in persian_names_list:\n",
        "#     print(word)\n",
        "#     corpus.remove(word)\n"
      ],
      "metadata": {
        "id": "zEeWAK2k5ndm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(corpus)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v3PJyZbp7jRc",
        "outputId": "de1a4681-bc20-4d3c-dff9-eb58c632be75"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "35316"
            ]
          },
          "metadata": {},
          "execution_count": 189
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# file = open('Nlp HW2/corpus_without_english_names.txt','w',encoding = 'utf-8')\n",
        "# for word in corpus:\n",
        "#   file.write(word)\n",
        "#   file.write(\"\\n\")\n",
        "# file.close()"
      ],
      "metadata": {
        "id": "w6geoVOd_NkK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#roles contains all the possible roles in our dictionary\n",
        "roles = [x[2] for x in hazm.words_list()]\n",
        "roles = list(set(itertools.chain(*roles)))\n",
        "roles"
      ],
      "metadata": {
        "id": "febyfjwQr0hc",
        "outputId": "fdc72596-7dfb-4c6e-d841-61c476fb694a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['AJC',\n",
              " 'NUM',\n",
              " 'CL',\n",
              " 'ZVR',\n",
              " 'POSTP',\n",
              " 'RES',\n",
              " 'V',\n",
              " 'COMP',\n",
              " 'PL',\n",
              " 'INT',\n",
              " 'PS',\n",
              " 'PR',\n",
              " 'P',\n",
              " 'NIN',\n",
              " 'PRO',\n",
              " 'N',\n",
              " 'CONJ',\n",
              " 'DET',\n",
              " 'ADV',\n",
              " 'AJ']"
            ]
          },
          "metadata": {},
          "execution_count": 191
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "lemmatizer = hazm.Lemmatizer()\n",
        "stemmer = hazm.Stemmer()"
      ],
      "metadata": {
        "id": "LZ0cqll7Zwdu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''this way of defining stop_words may not be usual,\n",
        "but the main purpose has been to eliminate disturbing words;\n",
        "such as: punctiation marks, pronouns and Persian\\Arabic grammatical signs'''\n",
        "\n",
        "stop_words = [\"ند\",\"یم\",\"ید\",\"ی\",\"تان\",\"ش\",\"ت\",\"شان\",\"مان\",\"م\",\"ها\",\"ات\",\"ان\",\"ین\",\"ون\",\"‌های\",\"ام\",\"ای\"]\n",
        "stop_words.extend([\".\",\"؛\",\",\",\":\",\"?\",\"!\",\"؟\",\"ْ\",\"ٌ\",\"ٍ\",\"ً\",\"ُ\",\"ِ\",\"َ\",\"ّ\",\"}\",\"ؤ\",\"{\",\"»\",\"«\",\"ٰ\",\"‌‌‌‌‌‌‌‌ٔ\",\"ء\",\"\\\"\",\"(\",\")\"])\n",
        "more_stopwords=[\"ها\",\"تر\",\"ترین\"]\n",
        "stop_words.extend(more_stopwords)"
      ],
      "metadata": {
        "id": "sUJqnvFEwe3v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "add_list = [\"ی\",\"تان\",\"ش\",\"ت\",\"شان\",\"مان\",\"م\",\"‌ها\"]\n",
        "adj_add_list = [\"تر\",\"ترین\"]\n",
        "corpu = []\n",
        "for word in hazm.words_list():\n",
        "  corpu.append(word[0])\n",
        "  corpu.append(lemmatizer.lemmatize(word[0]))\n",
        "  if (\"N\" in word[2]) or (\"AJC\" in word[2]) or (\"AJ\" in word[2]):\n",
        "    for ele in add_list:\n",
        "      corpu.append(word[0]+ele)\n",
        "  if (\"AJC\" in word[2]) or (\"AJ\" in word[2]):\n",
        "    for ele in adj_add_list:\n",
        "      corpu.append(word[0]+ele)\n",
        "\n",
        "corpu.append(\"آره\")\n",
        "corpu = list(set(corpu))\n",
        "len(corpu)"
      ],
      "metadata": {
        "id": "SlN7tsHwpAyJ",
        "outputId": "2a308bc8-4fdb-424a-bfcc-2c7ba8de8c36",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "336122"
            ]
          },
          "metadata": {},
          "execution_count": 194
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Main code"
      ],
      "metadata": {
        "id": "ezt1J32nymfM"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Config"
      ],
      "metadata": {
        "id": "uoztZzu6NAYp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#nlp is the dadmatools' pipeline used for lemmatization purpose only\n",
        "pips = 'lem'\n",
        "nlp = language.Pipeline(pips)\n",
        "\n",
        "print(nlp.analyze_pipes(pretty=True))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7ShvxXYdXJ0Y",
        "outputId": "0c2a30c5-fabd-440f-f695-719e86406fe1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model fa_tokenizer exists in /root/.pernlp/fa_tokenizer.pt\n",
            "Model fa_mwt exists in /root/.pernlp/fa_mwt.pt\n",
            "Model fa_lemmatizer exists in /root/.pernlp/fa_lemmatizer.pt\n",
            "\u001b[1m\n",
            "============================= Pipeline Overview =============================\u001b[0m\n",
            "\n",
            "#   Component   Assigns       Requires   Scores   Retokenizes\n",
            "-   ---------   -----------   --------   ------   -----------\n",
            "0   tokenizer                                     True       \n",
            "                                                             \n",
            "1   lemmatize   token.lemma                       False      \n",
            "\n",
            "\u001b[38;5;2m✔ No problems found.\u001b[0m\n",
            "{'summary': {'tokenizer': {'assigns': [], 'requires': [], 'scores': [], 'retokenizes': True}, 'lemmatize': {'assigns': ['token.lemma'], 'requires': [], 'scores': [], 'retokenizes': False}}, 'problems': {'tokenizer': [], 'lemmatize': []}, 'attrs': {'token.lemma': {'assigns': ['lemmatize'], 'requires': []}}}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#we use an args dictionary to initialize our model\n",
        "args = {\n",
        "    'corpus':corpu,\n",
        "    'pipe':nlp,\n",
        "    'stop_words':stop_words,\n",
        "    'normalizer':hazm.Normalizer(),\n",
        "    'lemmatizer':hazm.Lemmatizer(),\n",
        "    'POS_tagger':hazm.POSTagger(model='Nlp HW2/resources/postagger.model'),\n",
        "    'translator':None,\n",
        "    'word_tokenizer':hazm.WordTokenizer(),\n",
        "    'sent_tokenizer':hazm.SentenceTokenizer()\n",
        "}"
      ],
      "metadata": {
        "id": "qP0mg4wDOMIm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Model"
      ],
      "metadata": {
        "id": "6kvuGlYaNPYU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#as the name suggests, the class is used to spot and show the positin of foreign words within the text\n",
        "\n",
        "class foreign_word_detector():\n",
        "\n",
        "  def __init__(\n",
        "      self,\n",
        "      args\n",
        "      ):\n",
        "\n",
        "    self.corpus = args['corpus']\n",
        "    self.stop_words = args['stop_words']\n",
        "    self.lang_pipe = args['pipe']\n",
        "    self.normalizer = args['normalizer']\n",
        "    self.lemmatizer = args['lemmatizer']\n",
        "    self.translator = args['translator']\n",
        "    self.word_tokenizer = args['word_tokenizer']\n",
        "    self.sent_tokenizer = args['sent_tokenizer']\n",
        "    self.tagger = args['POS_tagger']\n",
        "\n",
        "#the following function is the main funtion of the model used for detecting the foreign words and their position\n",
        "  def detect(self, text):\n",
        "    sentenced = self.sent_tokenizer.tokenize(text)\n",
        "\n",
        "    normalized = [self.normalizer.normalize(sent) for sent in sentenced]\n",
        "    normalized = [sent.replace(\"\\u200cهای\",\"\") for sent in normalized]\n",
        "    normalized = [sent.replace(\"\\u200c\",\" \") for sent in normalized]\n",
        "    # print(f\"normalized:{normalized}\")\n",
        "    lemmed = [str(self.lang_pipe(sent)) for sent in normalized]\n",
        "    # print(f\"lemmed:{lemmed}\")\n",
        "    words_list = [self.word_tokenizer.tokenize(sent) for sent in lemmed]\n",
        "\n",
        "    final_words_list = []\n",
        "    for sent in words_list:\n",
        "      words_second_list = []\n",
        "      tags = self.tagger.tag(sent)\n",
        "      for i in range(len(tags)):\n",
        "        if tags[i][1] != \"V\":\n",
        "          words_second_list.append(tags[i][0])\n",
        "      final_words_list.append(words_second_list)\n",
        "\n",
        "    words_list = list(itertools.chain(*final_words_list))\n",
        "    words_list = [word for word in words_list if word not in self.stop_words]\n",
        "    #lemmed_words_list = [lemmatizer.lemmatize(word) for word in words_list]\n",
        "    # print(lemmed_words_list)\n",
        "\n",
        "    output = dict()\n",
        "    for word in words_list:\n",
        "      if word not in self.corpus:\n",
        "        output[word] = []\n",
        "        add_ind = 0\n",
        "\n",
        "        string = text\n",
        "        while word in string:\n",
        "\n",
        "          begin = string.find(word)\n",
        "          output[word].append((begin+add_ind,add_ind+begin+len(word)))\n",
        "\n",
        "          string = string[begin+len(word):]\n",
        "          add_ind+= begin+len(word)\n",
        "\n",
        "    return output\n",
        ""
      ],
      "metadata": {
        "id": "abisr-pQyq1p"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Test"
      ],
      "metadata": {
        "id": "Oyh-F0TrNX2y"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "####results"
      ],
      "metadata": {
        "id": "REhKcO_FRmuT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataframe = pd.read_csv(\"Nlp HW2/test_file.csv\")"
      ],
      "metadata": {
        "id": "l5ixR8F9R7fs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataframe.fillna(\"\",inplace=True)"
      ],
      "metadata": {
        "id": "dxk_qVqEJm-M"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#testing the model:\n",
        "detector = foreign_word_detector(args)\n",
        "true_count = 0\n",
        "tp_count = 0\n",
        "fp_count = 0\n",
        "fn_count = 0\n",
        "for i in range(dataframe.shape[0]):\n",
        "  sample = dataframe[\"sample\"][i]\n",
        "  words = dataframe[\"word\"][i].split()\n",
        "  results = detector.detect(sample).keys()\n",
        "  for word in results:\n",
        "    if word in words:\n",
        "      tp_count+=1\n",
        "    else:\n",
        "      fp_count+=1\n",
        "  true_count+=len(words)\n",
        "\n",
        "fn_count = true_count - tp_count\n",
        "\n",
        "print(f\"true_positive_count = {tp_count}\\nfalse_positive_count = {fp_count}\\nfalse_negative_count = {fn_count}\\nprecision = {tp_count/(tp_count+fp_count) :.2f}\\nrecall = {tp_count/(true_count):.2f}\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gzfBm5zm8hz3",
        "outputId": "6d53f6c8-a73e-40d4-e4ad-eadafa3f9a53"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "true_positive_count = 103\n",
            "false_positive_count = 8\n",
            "false_negative_count = 19\n",
            "precision = 0.93\n",
            "recall = 0.84\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "####user run"
      ],
      "metadata": {
        "id": "tljaM-3WRscu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#the run function, is an interactive way to map user_input to the detectors answer\n",
        "def run():\n",
        "  detector = foreign_word_detector(args)\n",
        "\n",
        "  print(\"enter your sample text.\\nenter an empty string if you want to end the process\")\n",
        "  while True :\n",
        "    text = input()\n",
        "    if text == \"\":\n",
        "      break\n",
        "\n",
        "    output = detector.detect(text)\n",
        "    print(output)\n"
      ],
      "metadata": {
        "id": "gFnE-4jRs3Hp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "run()"
      ],
      "metadata": {
        "id": "pvJUGGdQw1mH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##New idea"
      ],
      "metadata": {
        "id": "hr0gxCMLNfjp"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Translate and replace"
      ],
      "metadata": {
        "id": "sLE4BxvSEHIn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def change_english_words(text):\n",
        "  from googletrans import Translator\n",
        "  translator = Translator()\n",
        "  detector = foreign_word_detector(args)\n",
        "  output = detector.detect(text)\n",
        "  print(f\"Output: {output}\")\n",
        "  english_text=text\n",
        "  new_text=text\n",
        "  for word in output.keys():\n",
        "    # first_index=output[word][0]\n",
        "    # second_index=output[word][1]\n",
        "    english_form=translator.translate(word).text\n",
        "    persian_translation=translator.translate(english_form, dest='fa').text\n",
        "    persian_translation=persian_translation.split(\"-\")[0]\n",
        "    new_text=new_text.replace(word,persian_translation)\n",
        "  return new_text,english_text"
      ],
      "metadata": {
        "id": "BWa2xvzrNhFs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# text= \"تسک سختی بود ولی تو کانتریبیوشن خوبی داشتی تنکس .\"\n",
        "text= \"سلام من و فرزندت در گاردنمان همبرگر و استیک خوریدم و آن‌ ها خیلی دلیشز بودند .\"\n",
        "# text=\"من و فرندم نشستیم گیم آف ترونز دیدیم ولی اصلا به گرد پای برکینگ بد هم نمیرسید.\"\n",
        "translation,english_text=change_english_words(text)\n",
        "print(translation)\n",
        "# print(english_text)"
      ],
      "metadata": {
        "id": "qb3F74t0DUMA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2944005d-5f83-4e6e-93e2-eff56cfab67c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Output: {'گاردن': [(20, 25)], 'دلیشز': [(65, 70)]}\n",
            "سلام من و فرزندت در باغمان همبرگر و استیک خوریدم و آن‌ ها خیلی خوشمزه  بودند .\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Using phonetics"
      ],
      "metadata": {
        "id": "cNb5-GbmEOyn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#We import an English dictionary. Add the english words in \"words\" and the corresponding phonetics to \"phonics_list\".\n",
        "words = []\n",
        "phonics_list = []\n",
        "dict_file = open('Nlp HW2/cmudict.dict', 'r')\n",
        "with dict_file as f:\n",
        "    phonics = [line.rstrip('\\n') for line in f]\n",
        "    for p in phonics:\n",
        "        x = p.split(' ')\n",
        "        words.append(x[0])\n",
        "        phonics_list.append(' '.join(x[1:]))"
      ],
      "metadata": {
        "id": "_kzbejEP6GP7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "phons_unique = list(set(itertools.chain(*[ph.split(\" \") for ph in phonics_list])))\n",
        "phons_as_list = [ph.split(\" \") for ph in phonics_list]"
      ],
      "metadata": {
        "id": "zulC4Adu6upO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"Nlp HW2/phons_exchange.txt\",\"rb\") as file:\n",
        "  phons_dict = pickle.load(file)"
      ],
      "metadata": {
        "id": "vtwOxqizjC-X"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "phons_dict[\"NG\"] = (\"ن\")\n",
        "phons_dict[\"TH\"] = (\"ت\")\n",
        "phons_dict[\"AO2\"] = phons_dict[\"AO1\"]\n",
        "phons_dict[\"IH2\"] = (\"ای\",\"ی\")\n",
        "phons_dict[\"IH0\"] = phons_dict[\"IH2\"]\n",
        "phons_dict[\"IH1\"] = phons_dict[\"IH0\"]\n",
        "phons_dict[\"IY2\"] = (\"ای\",\"ی\")\n",
        "phons_dict[\"UH0\"] = (\"او\",\"و\")\n",
        "phons_dict[\"UH2\"] = phons_dict[\"UH0\"]\n",
        "phons_dict[\"UH1\"] = phons_dict[\"UH0\"]\n",
        "phons_dict[\"IY1\"] = phons_dict[\"IY2\"]\n",
        "phons_dict[\"IY0\"] = phons_dict[\"IY2\"]"
      ],
      "metadata": {
        "id": "6_UTCNuVjWlr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Converting our words to their persian correspondent with the use of phonetics in the \"persianized\"\n",
        "persianized_list = []\n",
        "persianized = []\n",
        "for x in phons_as_list:\n",
        "  word_list = []\n",
        "  count=0\n",
        "  for ph in x:\n",
        "    count+=1\n",
        "    if len(phons_dict[ph])>1:\n",
        "      if count==1:\n",
        "        word_list.append(phons_dict[ph][0])\n",
        "      else:\n",
        "        word_list.append(phons_dict[ph][1])\n",
        "    else:\n",
        "      word_list.append(phons_dict[ph][0])\n",
        "\n",
        "  persianized_list.append(word_list)\n",
        "  persianized.append(''.join(word_list))"
      ],
      "metadata": {
        "id": "TZQlhU9GjSJi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Checking some words from \"words\"\n",
        "words[41000:41010]"
      ],
      "metadata": {
        "id": "5_pSldPjknPo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dc84d250-2c64-42bc-85b2-feb984efcb7e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['fancier',\n",
              " 'fanciers',\n",
              " 'fancies',\n",
              " 'fanciest',\n",
              " 'fanciful',\n",
              " 'fancy',\n",
              " 'fandango',\n",
              " 'fandel',\n",
              " 'fandrich',\n",
              " 'fane']"
            ]
          },
          "metadata": {},
          "execution_count": 208
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Checking the corresponding persionized words from \"persionized\"\n",
        "persianized[41000:41010]"
      ],
      "metadata": {
        "id": "4mKUNAPmkdt9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c1e5519a-2846-4bb6-f1a7-8c1b94adfba8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['فنسیر',\n",
              " 'فنسیرز',\n",
              " 'فنسیز',\n",
              " 'فنسیست',\n",
              " 'فنسیفل',\n",
              " 'فنسی',\n",
              " 'فندنگو',\n",
              " 'فندل',\n",
              " 'فندریک',\n",
              " 'فین']"
            ]
          },
          "metadata": {},
          "execution_count": 209
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Checking if our persionized method works fine on some words.\n",
        "listt = ['contribution','thanks','computer','system','task','time','garden','burger','chop','fandrich','infinity']\n",
        "\n",
        "for i in range(len(words)):\n",
        "  if words[i] in listt:\n",
        "    print(words[i])\n",
        "    print(phons_as_list[i])\n",
        "    print(i)\n",
        "    print(persianized[i])\n",
        "    print(\"-----------------\")"
      ],
      "metadata": {
        "id": "Pm2B0nXskxX8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "18f2ae99-8734-4009-c360-02595acf0fb7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "burger\n",
            "['B', 'ER1', 'G', 'ER0']\n",
            "16631\n",
            "برگر\n",
            "-----------------\n",
            "chop\n",
            "['CH', 'AA1', 'P']\n",
            "21283\n",
            "چاپ\n",
            "-----------------\n",
            "computer\n",
            "['K', 'AH0', 'M', 'P', 'Y', 'UW1', 'T', 'ER0']\n",
            "23974\n",
            "کمپیوتر\n",
            "-----------------\n",
            "contribution\n",
            "['K', 'AA2', 'N', 'T', 'R', 'AH0', 'B', 'Y', 'UW1', 'SH', 'AH0', 'N']\n",
            "25006\n",
            "کانتربیوشن\n",
            "-----------------\n",
            "fandrich\n",
            "['F', 'AE1', 'N', 'D', 'R', 'IH0', 'K']\n",
            "41008\n",
            "فندریک\n",
            "-----------------\n",
            "garden\n",
            "['G', 'AA1', 'R', 'D', 'AH0', 'N']\n",
            "46458\n",
            "گاردن\n",
            "-----------------\n",
            "infinity\n",
            "['IH2', 'N', 'F', 'IH1', 'N', 'IH0', 'T', 'IY0']\n",
            "59556\n",
            "اینفینیتی\n",
            "-----------------\n",
            "system\n",
            "['S', 'IH1', 'S', 'T', 'AH0', 'M']\n",
            "118902\n",
            "سیستم\n",
            "-----------------\n",
            "task\n",
            "['T', 'AE1', 'S', 'K']\n",
            "119638\n",
            "تسک\n",
            "-----------------\n",
            "thanks\n",
            "['TH', 'AE1', 'NG', 'K', 'S']\n",
            "120780\n",
            "تنکس\n",
            "-----------------\n",
            "time\n",
            "['T', 'AY1', 'M']\n",
            "121710\n",
            "تایم\n",
            "-----------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#phons_dict contains the Persian equivalent of English phonetics\n",
        "phons_dict"
      ],
      "metadata": {
        "id": "4Gz8wC5fCZrn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "81790b0b-4fad-4fc7-b282-148f037efba6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'EH0': ('ا', ''),\n",
              " 'ER2': ('ار', 'ر'),\n",
              " 'old': ('',),\n",
              " 'EY0': ('ای', 'ی'),\n",
              " 'IY2': ('ای', 'ی'),\n",
              " 'name': ('',),\n",
              " 'S': ('س',),\n",
              " 'CH': ('چ',),\n",
              " 'K': ('ک',),\n",
              " 'UW2': ('یو',),\n",
              " 'AA0': ('آ', 'ا'),\n",
              " 'AY2': ('آی', 'ای'),\n",
              " 'OY0': ('اوی', 'وی'),\n",
              " 'AA1': ('آ', 'ا'),\n",
              " 'L': ('ل',),\n",
              " 'UH0': ('او', 'و'),\n",
              " 'B': ('ب',),\n",
              " 'F': ('ف',),\n",
              " 'OY1': ('اوی', 'وی'),\n",
              " 'abbrev': ('',),\n",
              " 'UH2': ('او', 'و'),\n",
              " 'D': ('د',),\n",
              " 'JH': ('ج',),\n",
              " 'W': ('و',),\n",
              " 'G': ('گ',),\n",
              " 'IH0': ('ای', 'ی'),\n",
              " 'UH1': ('او', 'و'),\n",
              " 'french': ('',),\n",
              " 'Z': ('ز',),\n",
              " 'AA2': ('آ', 'ا'),\n",
              " 'OY2': ('اوی', 'وی'),\n",
              " 'SH': ('ش',),\n",
              " 'AH0': ('ا', ''),\n",
              " 'foreign': ('',),\n",
              " 'UW0': ('یو',),\n",
              " 'AH1': ('ا', ''),\n",
              " 'IH1': ('ای', 'ی'),\n",
              " 'EY2': ('ای', 'ی'),\n",
              " 'AH2': ('ا', ''),\n",
              " 'OW2': ('او', 'و'),\n",
              " 'Y': ('ی',),\n",
              " 'AE2': ('',),\n",
              " 'DH': ('ض',),\n",
              " 'P': ('پ',),\n",
              " 'ER1': ('ار', 'ر'),\n",
              " 'AW2': ('او',),\n",
              " '#': ('',),\n",
              " 'EH1': ('ا', ''),\n",
              " 'AE1': ('',),\n",
              " 'EH2': ('ا', ''),\n",
              " 'N': ('ن',),\n",
              " 'ZH': ('ز',),\n",
              " 'OW1': ('او', 'و'),\n",
              " 'UW1': ('او', 'و'),\n",
              " 'AY0': ('آی', 'ای'),\n",
              " 'AY1': ('آی', 'ای'),\n",
              " 'AE0': ('',),\n",
              " 'NG': 'ن',\n",
              " 'AO1': ('او', 'و'),\n",
              " 'OW0': ('او', 'و'),\n",
              " 'HH': ('ه',),\n",
              " 'AW0': ('او',),\n",
              " 'TH': 'ت',\n",
              " 'IY0': ('ای', 'ی'),\n",
              " 'AO0': ('او', 'و'),\n",
              " 'AW1': ('او',),\n",
              " 'R': ('ر',),\n",
              " 'AO2': ('او', 'و'),\n",
              " 'M': ('م',),\n",
              " 'T': ('ت',),\n",
              " 'IH2': ('ای', 'ی'),\n",
              " 'V': ('و',),\n",
              " 'ER0': ('ار', 'ر'),\n",
              " 'IY1': ('ای', 'ی'),\n",
              " 'EY1': ('ای', 'ی')}"
            ]
          },
          "metadata": {},
          "execution_count": 211
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U strsimpy\n",
        "import pandas as pd\n",
        "from strsimpy.weighted_levenshtein import WeightedLevenshtein\n",
        "\n",
        "\n",
        "def insertion_cost(char):\n",
        "  if char==\"ا\":\n",
        "    return 0\n",
        "  return 1.0\n",
        "\n",
        "\n",
        "def deletion_cost(char):\n",
        "  if char==\"ا\":\n",
        "    return 0\n",
        "  return 1.0\n",
        "\n",
        "\n",
        "def substitution_cost(char_a, char_b):\n",
        "  if char_a==\"ا\" and char_b==\"آ\":\n",
        "    return 0\n",
        "  return 2.0\n",
        "\n",
        "weighted_levenshtein = WeightedLevenshtein(\n",
        "    substitution_cost_fn=substitution_cost,\n",
        "    insertion_cost_fn=insertion_cost,\n",
        "    deletion_cost_fn=deletion_cost)"
      ],
      "metadata": {
        "id": "wqRR8JTWpC3Y",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "50464509-e3c3-4ac9-b9d1-818d87e69030"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: strsimpy in /usr/local/lib/python3.9/dist-packages (0.2.1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "w=\"هاردی\"\n",
        "for i,word in enumerate(persianized):\n",
        "  if word==w:\n",
        "    print(word)\n",
        "    print(words[i])"
      ],
      "metadata": {
        "id": "LVtELnALuGxP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bcc8896c-fa52-4a8c-af3f-ad1a2ae68f34"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "هاردی\n",
            "hardee\n",
            "هاردی\n",
            "hardey\n",
            "هاردی\n",
            "hardie\n",
            "هاردی\n",
            "hardy\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "w=\"هاردی\"\n",
        "for i in hazm.words_list():\n",
        "  if i[0]==w:\n",
        "    print(i)"
      ],
      "metadata": {
        "id": "uuR5VnDL37ou",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f409afb0-6fb7-4e28-aebb-0e250cd85a80"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "('هاردی', 4, ('N',))\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "w1=\"کامپیوتر\"\n",
        "w2=\"کمپیوتر\"\n",
        "for i in persianized:\n",
        "  if i==w2:\n",
        "    print(i)\n",
        "weighted_levenshtein.distance(w1,w2)"
      ],
      "metadata": {
        "id": "Di9oMmfptFqz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d96114d4-5746-4586-9414-1536d3bd1687"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "کمپیوتر\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {},
          "execution_count": 215
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from hazm import *\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "from googletrans import Translator\n",
        "\n",
        "# Example sentences in Persian\n",
        "''' For words that are both Persian and English, we use the PickSentence method that chooses between two options. First option is the persian version and the second\n",
        "option is the english version. We check which sentence makes more sense and choose that sentence according to cosine similarity.\n",
        "'''\n",
        "intersected = list(set(persianized) & set(corpu))\n",
        "args = {\n",
        "    'corpus':corpu,\n",
        "    'pipe':nlp,\n",
        "    'stop_words':stop_words,\n",
        "    'intersected':intersected,\n",
        "    'normalizer':hazm.Normalizer(),\n",
        "    'lemmatizer':hazm.Lemmatizer(),\n",
        "    'POS_tagger':hazm.POSTagger(model='Nlp HW2/resources/postagger.model'),\n",
        "    'translator':Translator(),\n",
        "    'word_tokenizer':hazm.WordTokenizer(),\n",
        "    'sent_tokenizer':hazm.SentenceTokenizer()\n",
        "}"
      ],
      "metadata": {
        "id": "0O6tWgmiaAk-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#as the name suggests, the class is used to spot and show the positin of foreign words within the text\n",
        "class phonetics_foreign_word_detector():\n",
        "\n",
        "\n",
        "  def __init__(\n",
        "      self,\n",
        "      args\n",
        "      ):\n",
        "\n",
        "    self.corpus = args['corpus']\n",
        "    self.stop_words = args['stop_words']\n",
        "    self.intersected = args['intersected']\n",
        "    self.lang_pipe = args['pipe']\n",
        "    self.normalizer = args['normalizer']\n",
        "    self.lemmatizer = args['lemmatizer']\n",
        "    self.translator = args['translator']\n",
        "    self.word_tokenizer = args['word_tokenizer']\n",
        "    self.sent_tokenizer = args['sent_tokenizer']\n",
        "    self.tagger = args['POS_tagger']\n",
        "\n",
        "  def is_foreign_word(self,word):\n",
        "    for english_word in persianized:\n",
        "      if weighted_levenshtein.distance(word,english_word)==0:\n",
        "        return True\n",
        "    return False\n",
        "\n",
        "\n",
        "#the following function is the main funtion of the model used for detecting the foreign words and their position\n",
        "  def detect(self, text):\n",
        "    sentenced = self.sent_tokenizer.tokenize(text)\n",
        "\n",
        "    normalized = [self.normalizer.normalize(sent) for sent in sentenced]\n",
        "    normalized = [sent.replace(\"\\u200cهای\",\"\") for sent in normalized]\n",
        "    normalized = [sent.replace(\"\\u200c\",\" \") for sent in normalized]\n",
        "    # print(f\"normalized:{normalized}\")\n",
        "    lemmed = [str(self.lang_pipe(sent)) for sent in normalized]\n",
        "    # print(f\"lemmed:{lemmed}\")\n",
        "    words_list = [self.word_tokenizer.tokenize(sent) for sent in lemmed]\n",
        "\n",
        "    final_words_list = []\n",
        "    for sent in words_list:\n",
        "      words_second_list = []\n",
        "      tags = self.tagger.tag(sent)\n",
        "      for i in range(len(tags)):\n",
        "        if tags[i][1] != \"V\":\n",
        "          words_second_list.append(tags[i][0])\n",
        "      final_words_list.append(words_second_list)\n",
        "\n",
        "    words_list = list(itertools.chain(*final_words_list))\n",
        "    words_list = [word for word in words_list if word not in self.stop_words]\n",
        "    #lemmed_words_list = [lemmatizer.lemmatize(word) for word in words_list]\n",
        "    # print(lemmed_words_list)\n",
        "\n",
        "    output = dict()\n",
        "    for word in words_list:\n",
        "      if word in self.intersected:\n",
        "        option1 = word\n",
        "\n",
        "        english_form=self.translator.translate(option1).text\n",
        "        option2=self.translator.translate(english_form, dest='fa').text\n",
        "\n",
        "        sentence1= text\n",
        "        sentence2 = text.replace(option1,option2)\n",
        "\n",
        "        _,chosen_word = self.PickSentence(sentence1, sentence2, option1,option2)\n",
        "\n",
        "        if chosen_word == option2:\n",
        "          output[word] = []\n",
        "          add_ind = 0\n",
        "\n",
        "          string = text\n",
        "          while word in string:\n",
        "\n",
        "            begin = string.find(word)\n",
        "            output[word].append((begin+add_ind,add_ind+begin+len(word)))\n",
        "\n",
        "            string = string[begin+len(word):]\n",
        "            add_ind+= begin+len(word)\n",
        "\n",
        "      else:\n",
        "        if self.is_foreign_word(word):\n",
        "          output[word] = []\n",
        "          add_ind = 0\n",
        "\n",
        "          string = text\n",
        "          while word in string:\n",
        "\n",
        "            begin = string.find(word)\n",
        "            output[word].append((begin+add_ind,add_ind+begin+len(word)))\n",
        "\n",
        "            string = string[begin+len(word):]\n",
        "            add_ind+= begin+len(word)\n",
        "\n",
        "    return output\n",
        "\n",
        "  def PickSentence(self,sentence1, sentence2, option1,option2):\n",
        "\n",
        "    '''\n",
        "    sentence1 = \"من گرسنه هستم. آب میخواهم.\"\n",
        "    sentence2 = \"من تشنه هستم. آب میخواهم.\"\n",
        "\n",
        "    Candidate words\n",
        "    option1 = \"گرسنه\"\n",
        "    option2 = \"تشنه\"\n",
        "    '''\n",
        "\n",
        "    # Tokenize the sentences\n",
        "    tokens1 = word_tokenize(sentence1)\n",
        "    tokens2 = word_tokenize(sentence2)\n",
        "\n",
        "    # Identify the blank space in each sentence\n",
        "    blank_index1 = tokens1.index(option1)\n",
        "    blank_index2 = tokens2.index(option2)\n",
        "\n",
        "    # Contextualize each sentence without the blank space\n",
        "    context_tokens1 = tokens1[:blank_index1] + [\"____\"] + tokens1[blank_index1+1:]\n",
        "    context1 = ' '.join(context_tokens1)\n",
        "    context_tokens2 = tokens2[:blank_index2] + [\"____\"] + tokens2[blank_index2+1:]\n",
        "    context2 = ' '.join(context_tokens2)\n",
        "\n",
        "    # Use a language model to predict which sentence is more likely\n",
        "    normalizer = Normalizer()\n",
        "    doc1 = normalizer.normalize(sentence1)\n",
        "    context1 = normalizer.normalize(context1)\n",
        "    doc2 = normalizer.normalize(sentence2)\n",
        "    context2 = normalizer.normalize(context2)\n",
        "\n",
        "    stemmer = Stemmer()\n",
        "    doc_stemmed1 = ' '.join([stemmer.stem(word) for word in word_tokenize(doc1)])\n",
        "    context_stemmed1 = ' '.join([stemmer.stem(word) for word in word_tokenize(context1)])\n",
        "    doc_stemmed2 = ' '.join([stemmer.stem(word) for word in word_tokenize(doc2)])\n",
        "    context_stemmed2 = ' '.join([stemmer.stem(word) for word in word_tokenize(context2)])\n",
        "\n",
        "    # Calculate cosine similarity score for each option\n",
        "    vectorizer = TfidfVectorizer(token_pattern=r\"(?u)\\b\\w+\\b\")\n",
        "    X = vectorizer.fit_transform([doc_stemmed1, context_stemmed1, doc_stemmed2, context_stemmed2])\n",
        "    cosine_similarities1 = cosine_similarity(X)[0][1]\n",
        "    cosine_similarities2 = cosine_similarity(X)[2][3]\n",
        "\n",
        "    # Choose the sentence with the higher score\n",
        "    if cosine_similarities1 > cosine_similarities2:\n",
        "        chosen_sentence = sentence1\n",
        "        chosenword=option1\n",
        "\n",
        "    else:\n",
        "        chosen_sentence = sentence2\n",
        "        chosenword=option2\n",
        "\n",
        "    return chosen_sentence, chosenword"
      ],
      "metadata": {
        "id": "nP6htTpRniww"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#the run function, is an interactive way to map user_input to the detectors answer\n",
        "def phonetics_run():\n",
        "  detector = phonetics_foreign_word_detector(args)\n",
        "\n",
        "  print(\"enter your sample text.\\nenter an empty string if you want to end the process\")\n",
        "  while True :\n",
        "    text = input()\n",
        "    if text == \"\":\n",
        "      break\n",
        "\n",
        "    output = detector.detect(text)\n",
        "    print(output)\n"
      ],
      "metadata": {
        "id": "2kHEIVf8pl97"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "phonetics_run()"
      ],
      "metadata": {
        "id": "GDGwD8YCpsSc"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}