nlp/101/nlp_getting_started.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "nlp getting started",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/manishiitg/ML_Experments/blob/master/nlp/101/nlp_getting_started.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5nvjKLOQlUo7",
        "colab_type": "text"
      },
      "source": [
        "**Basic's of NLP to just try out and understand very basic concepts on NLP like tokenization, stopwords etc** "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FcxzQlxUMuap",
        "colab_type": "code",
        "outputId": "2749c12e-5f17-40ca-9fed-edc6f3f5650c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        }
      },
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "\n",
        "import spacy \n",
        "nlp = spacy.load(\"en_core_web_sm\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TBleKgedm2f8",
        "colab_type": "text"
      },
      "source": [
        "Using nltk and spacy library"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1hiPZIqNMyJm",
        "colab_type": "code",
        "outputId": "8af77098-458b-449d-887f-b196cfc1de4a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "# tokenize using nltk and spacy\n",
        "sentence = \"Natural language processing (NLP) is a subfield of linguistics, computer science, information engineering, and artificial intelligence concerned with the interactions between computers and human (natural) languages, in particular how to program computers to process and analyze large amounts of natural language data\"\n",
        "\n",
        "words = nltk.word_tokenize(sentence)\n",
        "print(words)\n",
        "\n",
        "\n",
        "doc = nlp(sentence)\n",
        "\n",
        "tokens = []\n",
        "\n",
        "for token in doc:\n",
        "    tokens.append(token.text)\n",
        "\n",
        "print(tokens)   "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['Natural', 'language', 'processing', '(', 'NLP', ')', 'is', 'a', 'subfield', 'of', 'linguistics', ',', 'computer', 'science', ',', 'information', 'engineering', ',', 'and', 'artificial', 'intelligence', 'concerned', 'with', 'the', 'interactions', 'between', 'computers', 'and', 'human', '(', 'natural', ')', 'languages', ',', 'in', 'particular', 'how', 'to', 'program', 'computers', 'to', 'process', 'and', 'analyze', 'large', 'amounts', 'of', 'natural', 'language', 'data']\n",
            "['Natural', 'language', 'processing', '(', 'NLP', ')', 'is', 'a', 'subfield', 'of', 'linguistics', ',', 'computer', 'science', ',', 'information', 'engineering', ',', 'and', 'artificial', 'intelligence', 'concerned', 'with', 'the', 'interactions', 'between', 'computers', 'and', 'human', '(', 'natural', ')', 'languages', ',', 'in', 'particular', 'how', 'to', 'program', 'computers', 'to', 'process', 'and', 'analyze', 'large', 'amounts', 'of', 'natural', 'language', 'data']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hPc1jTlYm_q0",
        "colab_type": "text"
      },
      "source": [
        "Comparing tokenize between spacy and nltk, comparing output for this same"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fg6huNfsO1F7",
        "colab_type": "code",
        "outputId": "6c33878b-bd6b-4453-e4d1-7ce6d3ec1731",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 103
        }
      },
      "source": [
        "# stemming nltk\n",
        "# import these modules \n",
        "from nltk.stem import PorterStemmer \n",
        "from nltk.tokenize import word_tokenize \n",
        "   \n",
        "ps = PorterStemmer() \n",
        "  \n",
        "# choose some words to be stemmed \n",
        "words = [\"program\", \"programs\", \"programer\", \"programing\", \"programers\"] \n",
        "  \n",
        "for w in words: \n",
        "    print(w, \" : \", ps.stem(w)) "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "program  :  program\n",
            "programs  :  program\n",
            "programer  :  program\n",
            "programing  :  program\n",
            "programers  :  program\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YLMvWS_SoBGu",
        "colab_type": "text"
      },
      "source": [
        "doing stemming on words. stemming is simply just chopping off prefix or suffix from words to get their base meaning "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KxZTu73OPhd6",
        "colab_type": "code",
        "outputId": "29dba3ea-1326-4748-9dd4-0d080bbeb68e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "#nltk lemmantization\n",
        "\n",
        "from nltk.stem import WordNetLemmatizer \n",
        "  \n",
        "lemmatizer = WordNetLemmatizer() \n",
        "  \n",
        "print(\"programers : \", lemmatizer.lemmatize(\"programers\"))\n",
        "print(\"am :\", lemmatizer.lemmatize(\"am\")) \n",
        "print(\"feet :\", lemmatizer.lemmatize(\"feet\")) "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "programers :  programers\n",
            "am : am\n",
            "feet : foot\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A9c76_RRoaJr",
        "colab_type": "text"
      },
      "source": [
        "doing lemmantization here, which again converting words to their bases forms but taking language features into consideration as well\n",
        "https://stackoverflow.com/questions/1787110/what-is-the-true-difference-between-lemmatization-vs-stemming"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y_5HzJzsZdP5",
        "colab_type": "code",
        "outputId": "e69133de-d3c5-4d25-b605-974c3145c76b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 883
        }
      },
      "source": [
        "# spacy lemma\n",
        "sentence = \"Natural language processing (NLP) is a subfield of linguistics, computer science, information engineering, and artificial intelligence concerned with the interactions between computers and human (natural) languages, in particular how to program computers to process and analyze large amounts of natural language data\"\n",
        "\n",
        "doc = nlp(sentence)\n",
        "\n",
        "for token in doc:\n",
        "    print(token.text , \":\" , token.lemma_)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Natural : natural\n",
            "language : language\n",
            "processing : processing\n",
            "( : (\n",
            "NLP : NLP\n",
            ") : )\n",
            "is : be\n",
            "a : a\n",
            "subfield : subfield\n",
            "of : of\n",
            "linguistics : linguistic\n",
            ", : ,\n",
            "computer : computer\n",
            "science : science\n",
            ", : ,\n",
            "information : information\n",
            "engineering : engineering\n",
            ", : ,\n",
            "and : and\n",
            "artificial : artificial\n",
            "intelligence : intelligence\n",
            "concerned : concern\n",
            "with : with\n",
            "the : the\n",
            "interactions : interaction\n",
            "between : between\n",
            "computers : computer\n",
            "and : and\n",
            "human : human\n",
            "( : (\n",
            "natural : natural\n",
            ") : )\n",
            "languages : language\n",
            ", : ,\n",
            "in : in\n",
            "particular : particular\n",
            "how : how\n",
            "to : to\n",
            "program : program\n",
            "computers : computer\n",
            "to : to\n",
            "process : process\n",
            "and : and\n",
            "analyze : analyze\n",
            "large : large\n",
            "amounts : amount\n",
            "of : of\n",
            "natural : natural\n",
            "language : language\n",
            "data : datum\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D_bDm4n8aYp2",
        "colab_type": "code",
        "outputId": "1aa99c4e-9291-4ea4-88c5-2594873187a9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 921
        }
      },
      "source": [
        "# pos tagging using nltk and spacy\n",
        "\n",
        "sentence = \"Natural language processing (NLP) is a subfield of linguistics, computer science, information engineering, and artificial intelligence concerned with the interactions between computers and human (natural) languages, in particular how to program computers to process and analyze large amounts of natural language data\"\n",
        "\n",
        "print(nltk.pos_tag(nltk.word_tokenize(sentence)))\n",
        "\n",
        "doc = nlp(sentence)\n",
        "\n",
        "for token in doc:\n",
        "    print(token.text , \":\" , token.lemma_, token.pos_, token.tag_)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('Natural', 'JJ'), ('language', 'NN'), ('processing', 'NN'), ('(', '('), ('NLP', 'NNP'), (')', ')'), ('is', 'VBZ'), ('a', 'DT'), ('subfield', 'NN'), ('of', 'IN'), ('linguistics', 'NNS'), (',', ','), ('computer', 'NN'), ('science', 'NN'), (',', ','), ('information', 'NN'), ('engineering', 'NN'), (',', ','), ('and', 'CC'), ('artificial', 'JJ'), ('intelligence', 'NN'), ('concerned', 'VBN'), ('with', 'IN'), ('the', 'DT'), ('interactions', 'NNS'), ('between', 'IN'), ('computers', 'NNS'), ('and', 'CC'), ('human', 'JJ'), ('(', '('), ('natural', 'JJ'), (')', ')'), ('languages', 'NNS'), (',', ','), ('in', 'IN'), ('particular', 'JJ'), ('how', 'WRB'), ('to', 'TO'), ('program', 'NN'), ('computers', 'NNS'), ('to', 'TO'), ('process', 'VB'), ('and', 'CC'), ('analyze', 'VB'), ('large', 'JJ'), ('amounts', 'NNS'), ('of', 'IN'), ('natural', 'JJ'), ('language', 'NN'), ('data', 'NNS')]\n",
            "Natural : natural ADJ JJ\n",
            "language : language NOUN NN\n",
            "processing : processing NOUN NN\n",
            "( : ( PUNCT -LRB-\n",
            "NLP : NLP PROPN NNP\n",
            ") : ) PUNCT -RRB-\n",
            "is : be VERB VBZ\n",
            "a : a DET DT\n",
            "subfield : subfield NOUN NN\n",
            "of : of ADP IN\n",
            "linguistics : linguistic NOUN NNS\n",
            ", : , PUNCT ,\n",
            "computer : computer NOUN NN\n",
            "science : science NOUN NN\n",
            ", : , PUNCT ,\n",
            "information : information NOUN NN\n",
            "engineering : engineering NOUN NN\n",
            ", : , PUNCT ,\n",
            "and : and CCONJ CC\n",
            "artificial : artificial ADJ JJ\n",
            "intelligence : intelligence NOUN NN\n",
            "concerned : concern VERB VBN\n",
            "with : with ADP IN\n",
            "the : the DET DT\n",
            "interactions : interaction NOUN NNS\n",
            "between : between ADP IN\n",
            "computers : computer NOUN NNS\n",
            "and : and CCONJ CC\n",
            "human : human NOUN NN\n",
            "( : ( PUNCT -LRB-\n",
            "natural : natural ADJ JJ\n",
            ") : ) PUNCT -RRB-\n",
            "languages : language NOUN NNS\n",
            ", : , PUNCT ,\n",
            "in : in ADP IN\n",
            "particular : particular ADJ JJ\n",
            "how : how ADV WRB\n",
            "to : to PART TO\n",
            "program : program NOUN NN\n",
            "computers : computer NOUN NNS\n",
            "to : to PART TO\n",
            "process : process VERB VB\n",
            "and : and CCONJ CC\n",
            "analyze : analyze VERB VB\n",
            "large : large ADJ JJ\n",
            "amounts : amount NOUN NNS\n",
            "of : of ADP IN\n",
            "natural : natural ADJ JJ\n",
            "language : language NOUN NN\n",
            "data : datum NOUN NNS\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r8DHCxHVoqjq",
        "colab_type": "text"
      },
      "source": [
        "POS tagging means to exract specific words sentenses and tag them like adjective, verb, noun etc "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qGK86f-EKk8I",
        "colab_type": "code",
        "outputId": "35db4110-7943-4a4b-def7-c47450fbec77",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "source": [
        "# simple stop words\n",
        "\n",
        "\n",
        "from nltk.corpus import stopwords\n",
        "print(stopwords.words(\"english\"))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn', \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn', \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", 'won', \"won't\", 'wouldn', \"wouldn't\"]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LGln5DS0o4iX",
        "colab_type": "text"
      },
      "source": [
        "Stop words are simply words, which are not important for doing nlp tasks. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GDSyWizPLT7I",
        "colab_type": "code",
        "outputId": "ac7a232b-e7e6-455e-efa9-b3f11030ea4e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "source": [
        "# nltk removing stop words\n",
        "\n",
        "\n",
        "stop_words = stopwords.words(\"english\")\n",
        "\n",
        "newStopWords = ['stopWord1','stopWord2']  # optional just to know we can do this\n",
        "stop_words.extend(newStopWords)\n",
        "\n",
        "sentence = \"Natural language processing (NLP) is a subfield of linguistics, computer science, information engineering, and artificial intelligence concerned with the interactions between computers and human (natural) languages, in particular how to program computers to process and analyze large amounts of natural language data\"\n",
        "\n",
        "words = nltk.word_tokenize(sentence)\n",
        "without_stop_words = [word for word in words if not word in stop_words]\n",
        "print(without_stop_words)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "['Natural', 'language', 'processing', '(', 'NLP', ')', 'subfield', 'linguistics', ',', 'computer', 'science', ',', 'information', 'engineering', ',', 'artificial', 'intelligence', 'concerned', 'interactions', 'computers', 'human', '(', 'natural', ')', 'languages', ',', 'particular', 'program', 'computers', 'process', 'analyze', 'large', 'amounts', 'natural', 'language', 'data']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5vuW1zhQLIT-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# simple stop words in spacy\n"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}