text-preprocessing .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:31.815182Z",
     "iopub.status.busy": "2022-12-28T06:53:31.814450Z",
     "iopub.status.idle": "2022-12-28T06:53:31.849496Z",
     "shell.execute_reply": "2022-12-28T06:53:31.848603Z",
     "shell.execute_reply.started": "2022-12-28T06:53:31.815046Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/imdb-dataset-of-50k-movie-reviews/IMDB Dataset.csv\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:31.851953Z",
     "iopub.status.busy": "2022-12-28T06:53:31.850986Z",
     "iopub.status.idle": "2022-12-28T06:53:33.276193Z",
     "shell.execute_reply": "2022-12-28T06:53:33.274530Z",
     "shell.execute_reply.started": "2022-12-28T06:53:31.851901Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('/kaggle/input/imdb-dataset-of-50k-movie-reviews/IMDB Dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.278132Z",
     "iopub.status.busy": "2022-12-28T06:53:33.277861Z",
     "iopub.status.idle": "2022-12-28T06:53:33.286150Z",
     "shell.execute_reply": "2022-12-28T06:53:33.285037Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.278088Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 2)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.288908Z",
     "iopub.status.busy": "2022-12-28T06:53:33.288659Z",
     "iopub.status.idle": "2022-12-28T06:53:33.323965Z",
     "shell.execute_reply": "2022-12-28T06:53:33.323155Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.288875Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>One of the other reviewers has mentioned that ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A wonderful little production. &lt;br /&gt;&lt;br /&gt;The...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I thought this was a wonderful way to spend ti...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Basically there's a family where a little boy ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Petter Mattei's \"Love in the Time of Money\" is...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review sentiment\n",
       "0  One of the other reviewers has mentioned that ...  positive\n",
       "1  A wonderful little production. <br /><br />The...  positive\n",
       "2  I thought this was a wonderful way to spend ti...  positive\n",
       "3  Basically there's a family where a little boy ...  negative\n",
       "4  Petter Mattei's \"Love in the Time of Money\" is...  positive"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.325079Z",
     "iopub.status.busy": "2022-12-28T06:53:33.324893Z",
     "iopub.status.idle": "2022-12-28T06:53:33.332647Z",
     "shell.execute_reply": "2022-12-28T06:53:33.331887Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.325054Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"basically there's a family where a little boy (jake) thinks there's a zombie in his closet & his parents are fighting all the time.<br /><br />this movie is slower than a soap opera... and suddenly, jake decides to become rambo and kill the zombie.<br /><br />ok, first of all when you're going to make a film you must decide if its a thriller or a drama! as a drama the movie is watchable. parents are divorcing & arguing like in real life. and then we have jake with his closet which totally ruins all the film! i expected to see a boogeyman similar movie, and instead i watched a drama with some meaningless thriller spots.<br /><br />3 out of 10 just for the well playing parents & descent dialogs. as for the shots with jake: just ignore them.\""
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['review'][3].lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.333985Z",
     "iopub.status.busy": "2022-12-28T06:53:33.333755Z",
     "iopub.status.idle": "2022-12-28T06:53:33.464780Z",
     "shell.execute_reply": "2022-12-28T06:53:33.463554Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.333939Z"
    }
   },
   "outputs": [],
   "source": [
    "df['review'] = df['review'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.466301Z",
     "iopub.status.busy": "2022-12-28T06:53:33.466019Z",
     "iopub.status.idle": "2022-12-28T06:53:33.477828Z",
     "shell.execute_reply": "2022-12-28T06:53:33.477271Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.466234Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one of the other reviewers has mentioned that ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a wonderful little production. &lt;br /&gt;&lt;br /&gt;the...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>i thought this was a wonderful way to spend ti...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>basically there's a family where a little boy ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>petter mattei's \"love in the time of money\" is...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49995</th>\n",
       "      <td>i thought this movie did a down right good job...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49996</th>\n",
       "      <td>bad plot, bad dialogue, bad acting, idiotic di...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49997</th>\n",
       "      <td>i am a catholic taught in parochial elementary...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>i'm going to have to disagree with the previou...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>no one expects the star trek movies to be high...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  review sentiment\n",
       "0      one of the other reviewers has mentioned that ...  positive\n",
       "1      a wonderful little production. <br /><br />the...  positive\n",
       "2      i thought this was a wonderful way to spend ti...  positive\n",
       "3      basically there's a family where a little boy ...  negative\n",
       "4      petter mattei's \"love in the time of money\" is...  positive\n",
       "...                                                  ...       ...\n",
       "49995  i thought this movie did a down right good job...  positive\n",
       "49996  bad plot, bad dialogue, bad acting, idiotic di...  negative\n",
       "49997  i am a catholic taught in parochial elementary...  negative\n",
       "49998  i'm going to have to disagree with the previou...  negative\n",
       "49999  no one expects the star trek movies to be high...  negative\n",
       "\n",
       "[50000 rows x 2 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.479036Z",
     "iopub.status.busy": "2022-12-28T06:53:33.478847Z",
     "iopub.status.idle": "2022-12-28T06:53:33.490023Z",
     "shell.execute_reply": "2022-12-28T06:53:33.488923Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.479012Z"
    }
   },
   "outputs": [],
   "source": [
    "import re\n",
    "def remove_html_tags(text):\n",
    "    pattern = re.compile('<.*?>')\n",
    "    return pattern.sub(r'', text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.491885Z",
     "iopub.status.busy": "2022-12-28T06:53:33.491655Z",
     "iopub.status.idle": "2022-12-28T06:53:33.502601Z",
     "shell.execute_reply": "2022-12-28T06:53:33.501895Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.491849Z"
    }
   },
   "outputs": [],
   "source": [
    "text = \"<html><body><p> Movie 1</p><p> Actor - Aamir Khan</p><p> Click here to <a href='http://google.com'>download</a></p></body></html>\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.507391Z",
     "iopub.status.busy": "2022-12-28T06:53:33.507073Z",
     "iopub.status.idle": "2022-12-28T06:53:33.516743Z",
     "shell.execute_reply": "2022-12-28T06:53:33.515934Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.507345Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' Movie 1 Actor - Aamir Khan Click here to download'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "remove_html_tags(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.518328Z",
     "iopub.status.busy": "2022-12-28T06:53:33.517995Z",
     "iopub.status.idle": "2022-12-28T06:53:33.678914Z",
     "shell.execute_reply": "2022-12-28T06:53:33.677741Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.518291Z"
    }
   },
   "outputs": [],
   "source": [
    "df['review'] = df['review'].apply(remove_html_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.680820Z",
     "iopub.status.busy": "2022-12-28T06:53:33.680591Z",
     "iopub.status.idle": "2022-12-28T06:53:33.688803Z",
     "shell.execute_reply": "2022-12-28T06:53:33.687347Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.680794Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'probably my all-time favorite movie, a story of selflessness, sacrifice and dedication to a noble cause, but it\\'s not preachy or boring. it just never gets old, despite my having seen it some 15 or more times in the last 25 years. paul lukas\\' performance brings tears to my eyes, and bette davis, in one of her very few truly sympathetic roles, is a delight. the kids are, as grandma says, more like \"dressed-up midgets\" than children, but that only makes them more fun to watch. and the mother\\'s slow awakening to what\\'s happening in the world and under her own roof is believable and startling. if i had a dozen thumbs, they\\'d all be \"up\" for this movie.'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['review'][5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.690130Z",
     "iopub.status.busy": "2022-12-28T06:53:33.689913Z",
     "iopub.status.idle": "2022-12-28T06:53:33.702503Z",
     "shell.execute_reply": "2022-12-28T06:53:33.701286Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.690101Z"
    }
   },
   "outputs": [],
   "source": [
    "def remove_url(text):\n",
    "#      pattern = re.compile(r'https?://\\S+|www\\.\\S+')\n",
    "#     return pattern.sub(r'', text)\n",
    "#     return re.sub(r'http\\S+', '',text)\n",
    "    return re.sub(r'https?://\\S+|www\\.\\S+','',text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.705248Z",
     "iopub.status.busy": "2022-12-28T06:53:33.704883Z",
     "iopub.status.idle": "2022-12-28T06:53:33.713973Z",
     "shell.execute_reply": "2022-12-28T06:53:33.712792Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.705186Z"
    }
   },
   "outputs": [],
   "source": [
    "text1 = 'Check out my notebook https://www.kaggle.com/campusx/notebook8223fc1abb'\n",
    "text2 = 'Check out my notebook http://www.kaggle.com/campusx/notebook8223fc1abb'\n",
    "text3 = 'Google search here www.google.com'\n",
    "text4 = 'For notebook click https://www.kaggle.com/campusx/notebook8223fc1abb to search check www.google.com'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.715486Z",
     "iopub.status.busy": "2022-12-28T06:53:33.715173Z",
     "iopub.status.idle": "2022-12-28T06:53:33.728326Z",
     "shell.execute_reply": "2022-12-28T06:53:33.727326Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.715448Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'For notebook click  to search check '"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "remove_url(text4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.731258Z",
     "iopub.status.busy": "2022-12-28T06:53:33.730072Z",
     "iopub.status.idle": "2022-12-28T06:53:33.744292Z",
     "shell.execute_reply": "2022-12-28T06:53:33.743127Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.731137Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import string,time\n",
    "string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.745563Z",
     "iopub.status.busy": "2022-12-28T06:53:33.745346Z",
     "iopub.status.idle": "2022-12-28T06:53:33.754530Z",
     "shell.execute_reply": "2022-12-28T06:53:33.753175Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.745535Z"
    }
   },
   "outputs": [],
   "source": [
    "exclude = string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.756569Z",
     "iopub.status.busy": "2022-12-28T06:53:33.756243Z",
     "iopub.status.idle": "2022-12-28T06:53:33.766879Z",
     "shell.execute_reply": "2022-12-28T06:53:33.765026Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.756533Z"
    }
   },
   "outputs": [],
   "source": [
    "def remove_punc(text):\n",
    "    for char in exclude:\n",
    "        text=text.replace(char,\"\")\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.770157Z",
     "iopub.status.busy": "2022-12-28T06:53:33.769868Z",
     "iopub.status.idle": "2022-12-28T06:53:33.781691Z",
     "shell.execute_reply": "2022-12-28T06:53:33.780783Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.770115Z"
    }
   },
   "outputs": [],
   "source": [
    "text = 'string. With. Punctuation?'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.783492Z",
     "iopub.status.busy": "2022-12-28T06:53:33.783045Z",
     "iopub.status.idle": "2022-12-28T06:53:33.794930Z",
     "shell.execute_reply": "2022-12-28T06:53:33.794098Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.783453Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'string With Punctuation'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "remove_punc(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.796491Z",
     "iopub.status.busy": "2022-12-28T06:53:33.796191Z",
     "iopub.status.idle": "2022-12-28T06:53:33.808869Z",
     "shell.execute_reply": "2022-12-28T06:53:33.807305Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.796454Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "string With Punctuation\n",
      "54.81243133544922\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "print(remove_punc(text))\n",
    "time1 = time.time() - start\n",
    "print(time1*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.811020Z",
     "iopub.status.busy": "2022-12-28T06:53:33.810621Z",
     "iopub.status.idle": "2022-12-28T06:53:33.817554Z",
     "shell.execute_reply": "2022-12-28T06:53:33.816504Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.810982Z"
    }
   },
   "outputs": [],
   "source": [
    "def remove_punc1(text):\n",
    "    return text.translate(str.maketrans('', '', exclude))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.819565Z",
     "iopub.status.busy": "2022-12-28T06:53:33.819091Z",
     "iopub.status.idle": "2022-12-28T06:53:33.831153Z",
     "shell.execute_reply": "2022-12-28T06:53:33.830251Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.819530Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.806850433349609\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "remove_punc1(text)\n",
    "time2 = time.time() - start\n",
    "print(time2*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.832710Z",
     "iopub.status.busy": "2022-12-28T06:53:33.832498Z",
     "iopub.status.idle": "2022-12-28T06:53:33.844375Z",
     "shell.execute_reply": "2022-12-28T06:53:33.843647Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.832684Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.052539404553414"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time1/time2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.846196Z",
     "iopub.status.busy": "2022-12-28T06:53:33.845791Z",
     "iopub.status.idle": "2022-12-28T06:53:33.856707Z",
     "shell.execute_reply": "2022-12-28T06:53:33.855904Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.846157Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'probably my all-time favorite movie, a story of selflessness, sacrifice and dedication to a noble cause, but it\\'s not preachy or boring. it just never gets old, despite my having seen it some 15 or more times in the last 25 years. paul lukas\\' performance brings tears to my eyes, and bette davis, in one of her very few truly sympathetic roles, is a delight. the kids are, as grandma says, more like \"dressed-up midgets\" than children, but that only makes them more fun to watch. and the mother\\'s slow awakening to what\\'s happening in the world and under her own roof is believable and startling. if i had a dozen thumbs, they\\'d all be \"up\" for this movie.'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['review'][5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.858774Z",
     "iopub.status.busy": "2022-12-28T06:53:33.858376Z",
     "iopub.status.idle": "2022-12-28T06:53:33.869322Z",
     "shell.execute_reply": "2022-12-28T06:53:33.867906Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.858745Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'probably my alltime favorite movie a story of selflessness sacrifice and dedication to a noble cause but its not preachy or boring it just never gets old despite my having seen it some 15 or more times in the last 25 years paul lukas performance brings tears to my eyes and bette davis in one of her very few truly sympathetic roles is a delight the kids are as grandma says more like dressedup midgets than children but that only makes them more fun to watch and the mothers slow awakening to whats happening in the world and under her own roof is believable and startling if i had a dozen thumbs theyd all be up for this movie'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "remove_punc1(df['review'][5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.871834Z",
     "iopub.status.busy": "2022-12-28T06:53:33.871074Z",
     "iopub.status.idle": "2022-12-28T06:53:33.881005Z",
     "shell.execute_reply": "2022-12-28T06:53:33.880283Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.871796Z"
    }
   },
   "outputs": [],
   "source": [
    "chat_words={\"AAFAIK\": \"As Far As I Know\",\n",
    "\"AFK\": \"Away From Keyboard\",\n",
    "\"ASAP\": \"As Soon As Possible\",\n",
    "\"ATK\": \"At The Keyboard\",\n",
    "\"ATM\": \"At The Moment\",\n",
    "\"A3\": \"Anytime, Anywhere, Anyplace\",\n",
    "\"BAK\": \"Back At Keyboard\",\n",
    "\"BBL\": \"Be Back Later\",\n",
    "\"BBS\": \"Be Back Soon\",\n",
    "\"BFN\": \"Bye For Now\",\n",
    "\"B4N\": \"Bye For Now\",\n",
    "\"BRB\": \"Be Right Back\",\n",
    "\"BRT\": \"Be Right There\",\n",
    "\"BTW\": \"By The Way\",\n",
    "\"B4\": \"Before\",\n",
    "\"B4N\": \"Bye For Now\",\n",
    "\"CU\": \"See You\",\n",
    "\"CUL8R\": \"See You Later\",\n",
    "\"CYA\": \"See You\",\n",
    "\"FAQ\": \"Frequently Asked Questions\",\n",
    "\"FC\": \"Fingers Crossed\",\n",
    "\"FWIW\": \"For What It's Worth\",\n",
    "\"FYI\": \"For Your Information\",\n",
    "\"GAL\": \"Get A Life\",\n",
    "\"GG\": \"Good Game\",\n",
    "\"GN\": \"Good Night\",\n",
    "\"GMTA\": \"Great Minds Think Alike\",\n",
    "\"GR8\": \"Great!\",\n",
    "\"G9\": \"Genius\",\n",
    "\"IC\": \"I See\",\n",
    "\"ICQ\": \"I Seek you (also a chat program)\",\n",
    "\"ILU\": \"ILU: I Love You\",\n",
    "\"IMHO\": \"In My Honest/Humble Opinion\",\n",
    "\"IMO\": \"In My Opinion\",\n",
    "\"IOW\": \"In Other Words\",\n",
    "\"IRL\": \"In Real Life\",\n",
    "\"KISS\": \"Keep It Simple, Stupid\",\n",
    "\"LDR\": \"Long Distance Relationship\",\n",
    "\"LMAO\": \"Laugh My A.. Off\",\n",
    "\"LOL\": \"Laughing Out Loud\",\n",
    "\"LTNS\": \"Long Time No See\",\n",
    "\"L8R\": \"Later\",\n",
    "\"MTE\": \"My Thoughts Exactly\",\n",
    "\"M8\": \"ate\",\n",
    "\"NRN\": \"No Reply Necessary\",\n",
    "\"OIC\": \"Oh I See\",\n",
    "\"PITA\": \"Pain In The A..\",\n",
    "\"PRT\": \"Party\",\n",
    "\"PRW\": \"Parents Are Watching\",\n",
    "\"ROFL\": \"Rolling On The Floor Laughing\",\n",
    "\"ROFLOL\": \"Rolling On The Floor Laughing Out Loud\",\n",
    "\"ROTFLMAO\": \"Rolling On The Floor Laughing My A.. Off\",\n",
    "\"SK8\": \"Skate\",\n",
    "\"STATS\": \"Your sex and age\",\n",
    "\"ASL\": \"ge, Sex, Location\",\n",
    "\"THX\": \"Thank You\",\n",
    "\"TTFN\": \"Ta-Ta For Now!\",\n",
    "\"TTYL\": \"Talk To You Later\",\n",
    "\"U\": \"You\",\n",
    "\"U2\": \"You Too\",\n",
    "\"U4E\":\"Yours For Ever\",\n",
    "\"WB\": \"Welcome Back\",\n",
    "\"WTF\": \"What The F...\",\n",
    "\"WTG\": \"Way To Go!\",\n",
    "\"WUF\": \"Where Are You From?\",\n",
    "\"W8\": \"Wait...\",\n",
    "\"7K\": \"Sick:-D Laugher\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.886076Z",
     "iopub.status.busy": "2022-12-28T06:53:33.884882Z",
     "iopub.status.idle": "2022-12-28T06:53:33.901283Z",
     "shell.execute_reply": "2022-12-28T06:53:33.900669Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.886034Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'AAFAIK': 'As Far As I Know',\n",
       " 'AFK': 'Away From Keyboard',\n",
       " 'ASAP': 'As Soon As Possible',\n",
       " 'ATK': 'At The Keyboard',\n",
       " 'ATM': 'At The Moment',\n",
       " 'A3': 'Anytime, Anywhere, Anyplace',\n",
       " 'BAK': 'Back At Keyboard',\n",
       " 'BBL': 'Be Back Later',\n",
       " 'BBS': 'Be Back Soon',\n",
       " 'BFN': 'Bye For Now',\n",
       " 'B4N': 'Bye For Now',\n",
       " 'BRB': 'Be Right Back',\n",
       " 'BRT': 'Be Right There',\n",
       " 'BTW': 'By The Way',\n",
       " 'B4': 'Before',\n",
       " 'CU': 'See You',\n",
       " 'CUL8R': 'See You Later',\n",
       " 'CYA': 'See You',\n",
       " 'FAQ': 'Frequently Asked Questions',\n",
       " 'FC': 'Fingers Crossed',\n",
       " 'FWIW': \"For What It's Worth\",\n",
       " 'FYI': 'For Your Information',\n",
       " 'GAL': 'Get A Life',\n",
       " 'GG': 'Good Game',\n",
       " 'GN': 'Good Night',\n",
       " 'GMTA': 'Great Minds Think Alike',\n",
       " 'GR8': 'Great!',\n",
       " 'G9': 'Genius',\n",
       " 'IC': 'I See',\n",
       " 'ICQ': 'I Seek you (also a chat program)',\n",
       " 'ILU': 'ILU: I Love You',\n",
       " 'IMHO': 'In My Honest/Humble Opinion',\n",
       " 'IMO': 'In My Opinion',\n",
       " 'IOW': 'In Other Words',\n",
       " 'IRL': 'In Real Life',\n",
       " 'KISS': 'Keep It Simple, Stupid',\n",
       " 'LDR': 'Long Distance Relationship',\n",
       " 'LMAO': 'Laugh My A.. Off',\n",
       " 'LOL': 'Laughing Out Loud',\n",
       " 'LTNS': 'Long Time No See',\n",
       " 'L8R': 'Later',\n",
       " 'MTE': 'My Thoughts Exactly',\n",
       " 'M8': 'ate',\n",
       " 'NRN': 'No Reply Necessary',\n",
       " 'OIC': 'Oh I See',\n",
       " 'PITA': 'Pain In The A..',\n",
       " 'PRT': 'Party',\n",
       " 'PRW': 'Parents Are Watching',\n",
       " 'ROFL': 'Rolling On The Floor Laughing',\n",
       " 'ROFLOL': 'Rolling On The Floor Laughing Out Loud',\n",
       " 'ROTFLMAO': 'Rolling On The Floor Laughing My A.. Off',\n",
       " 'SK8': 'Skate',\n",
       " 'STATS': 'Your sex and age',\n",
       " 'ASL': 'ge, Sex, Location',\n",
       " 'THX': 'Thank You',\n",
       " 'TTFN': 'Ta-Ta For Now!',\n",
       " 'TTYL': 'Talk To You Later',\n",
       " 'U': 'You',\n",
       " 'U2': 'You Too',\n",
       " 'U4E': 'Yours For Ever',\n",
       " 'WB': 'Welcome Back',\n",
       " 'WTF': 'What The F...',\n",
       " 'WTG': 'Way To Go!',\n",
       " 'WUF': 'Where Are You From?',\n",
       " 'W8': 'Wait...',\n",
       " '7K': 'Sick:-D Laugher'}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chat_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.902885Z",
     "iopub.status.busy": "2022-12-28T06:53:33.902334Z",
     "iopub.status.idle": "2022-12-28T06:53:33.912685Z",
     "shell.execute_reply": "2022-12-28T06:53:33.911830Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.902853Z"
    }
   },
   "outputs": [],
   "source": [
    "def chat_conversion(text):\n",
    "    new_text = []\n",
    "    for w in text.split():\n",
    "        if w.upper() in chat_words:\n",
    "            new_text.append(chat_words[w.upper()])\n",
    "        else:\n",
    "            new_text.append(w)\n",
    "    return \" \".join(new_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.914384Z",
     "iopub.status.busy": "2022-12-28T06:53:33.913862Z",
     "iopub.status.idle": "2022-12-28T06:53:33.927538Z",
     "shell.execute_reply": "2022-12-28T06:53:33.925481Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.914346Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'In My Honest/Humble Opinion he is the best'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chat_conversion('IMHO he is the best')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.929889Z",
     "iopub.status.busy": "2022-12-28T06:53:33.928728Z",
     "iopub.status.idle": "2022-12-28T06:53:33.943790Z",
     "shell.execute_reply": "2022-12-28T06:53:33.942931Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.929797Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'For Your Information delhi is the capital of india'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chat_conversion('FYI delhi is the capital of india')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:33.945349Z",
     "iopub.status.busy": "2022-12-28T06:53:33.945055Z",
     "iopub.status.idle": "2022-12-28T06:53:35.834937Z",
     "shell.execute_reply": "2022-12-28T06:53:35.833645Z",
     "shell.execute_reply.started": "2022-12-28T06:53:33.945309Z"
    }
   },
   "outputs": [],
   "source": [
    "from textblob import TextBlob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:35.836922Z",
     "iopub.status.busy": "2022-12-28T06:53:35.836629Z",
     "iopub.status.idle": "2022-12-28T06:53:35.905366Z",
     "shell.execute_reply": "2022-12-28T06:53:35.904652Z",
     "shell.execute_reply.started": "2022-12-28T06:53:35.836883Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'certain conditions during several generations are modified in the same manner.'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "incorrect_text = 'ceertain conditionas duriing seveal ggenerations aree moodified in the saame maner.'\n",
    "\n",
    "textBlb = TextBlob(incorrect_text)\n",
    "\n",
    "textBlb.correct().string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:35.907114Z",
     "iopub.status.busy": "2022-12-28T06:53:35.906536Z",
     "iopub.status.idle": "2022-12-28T06:53:35.911750Z",
     "shell.execute_reply": "2022-12-28T06:53:35.910755Z",
     "shell.execute_reply.started": "2022-12-28T06:53:35.907074Z"
    }
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:35.914581Z",
     "iopub.status.busy": "2022-12-28T06:53:35.914086Z",
     "iopub.status.idle": "2022-12-28T06:53:35.939681Z",
     "shell.execute_reply": "2022-12-28T06:53:35.938810Z",
     "shell.execute_reply.started": "2022-12-28T06:53:35.914541Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i',\n",
       " 'me',\n",
       " 'my',\n",
       " 'myself',\n",
       " 'we',\n",
       " 'our',\n",
       " 'ours',\n",
       " 'ourselves',\n",
       " 'you',\n",
       " \"you're\",\n",
       " \"you've\",\n",
       " \"you'll\",\n",
       " \"you'd\",\n",
       " 'your',\n",
       " 'yours',\n",
       " 'yourself',\n",
       " 'yourselves',\n",
       " 'he',\n",
       " 'him',\n",
       " 'his',\n",
       " 'himself',\n",
       " 'she',\n",
       " \"she's\",\n",
       " 'her',\n",
       " 'hers',\n",
       " 'herself',\n",
       " 'it',\n",
       " \"it's\",\n",
       " 'its',\n",
       " 'itself',\n",
       " 'they',\n",
       " 'them',\n",
       " 'their',\n",
       " 'theirs',\n",
       " 'themselves',\n",
       " 'what',\n",
       " 'which',\n",
       " 'who',\n",
       " 'whom',\n",
       " 'this',\n",
       " 'that',\n",
       " \"that'll\",\n",
       " 'these',\n",
       " 'those',\n",
       " 'am',\n",
       " 'is',\n",
       " 'are',\n",
       " 'was',\n",
       " 'were',\n",
       " 'be',\n",
       " 'been',\n",
       " 'being',\n",
       " 'have',\n",
       " 'has',\n",
       " 'had',\n",
       " 'having',\n",
       " 'do',\n",
       " 'does',\n",
       " 'did',\n",
       " 'doing',\n",
       " 'a',\n",
       " 'an',\n",
       " 'the',\n",
       " 'and',\n",
       " 'but',\n",
       " 'if',\n",
       " 'or',\n",
       " 'because',\n",
       " 'as',\n",
       " 'until',\n",
       " 'while',\n",
       " 'of',\n",
       " 'at',\n",
       " 'by',\n",
       " 'for',\n",
       " 'with',\n",
       " 'about',\n",
       " 'against',\n",
       " 'between',\n",
       " 'into',\n",
       " 'through',\n",
       " 'during',\n",
       " 'before',\n",
       " 'after',\n",
       " 'above',\n",
       " 'below',\n",
       " 'to',\n",
       " 'from',\n",
       " 'up',\n",
       " 'down',\n",
       " 'in',\n",
       " 'out',\n",
       " 'on',\n",
       " 'off',\n",
       " 'over',\n",
       " 'under',\n",
       " 'again',\n",
       " 'further',\n",
       " 'then',\n",
       " 'once',\n",
       " 'here',\n",
       " 'there',\n",
       " 'when',\n",
       " 'where',\n",
       " 'why',\n",
       " 'how',\n",
       " 'all',\n",
       " 'any',\n",
       " 'both',\n",
       " 'each',\n",
       " 'few',\n",
       " 'more',\n",
       " 'most',\n",
       " 'other',\n",
       " 'some',\n",
       " 'such',\n",
       " 'no',\n",
       " 'nor',\n",
       " 'not',\n",
       " 'only',\n",
       " 'own',\n",
       " 'same',\n",
       " 'so',\n",
       " 'than',\n",
       " 'too',\n",
       " 'very',\n",
       " 's',\n",
       " 't',\n",
       " 'can',\n",
       " 'will',\n",
       " 'just',\n",
       " 'don',\n",
       " \"don't\",\n",
       " 'should',\n",
       " \"should've\",\n",
       " 'now',\n",
       " 'd',\n",
       " 'll',\n",
       " 'm',\n",
       " 'o',\n",
       " 're',\n",
       " 've',\n",
       " 'y',\n",
       " 'ain',\n",
       " 'aren',\n",
       " \"aren't\",\n",
       " 'couldn',\n",
       " \"couldn't\",\n",
       " 'didn',\n",
       " \"didn't\",\n",
       " 'doesn',\n",
       " \"doesn't\",\n",
       " 'hadn',\n",
       " \"hadn't\",\n",
       " 'hasn',\n",
       " \"hasn't\",\n",
       " 'haven',\n",
       " \"haven't\",\n",
       " 'isn',\n",
       " \"isn't\",\n",
       " 'ma',\n",
       " 'mightn',\n",
       " \"mightn't\",\n",
       " 'mustn',\n",
       " \"mustn't\",\n",
       " 'needn',\n",
       " \"needn't\",\n",
       " 'shan',\n",
       " \"shan't\",\n",
       " 'shouldn',\n",
       " \"shouldn't\",\n",
       " 'wasn',\n",
       " \"wasn't\",\n",
       " 'weren',\n",
       " \"weren't\",\n",
       " 'won',\n",
       " \"won't\",\n",
       " 'wouldn',\n",
       " \"wouldn't\"]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:35.941481Z",
     "iopub.status.busy": "2022-12-28T06:53:35.941145Z",
     "iopub.status.idle": "2022-12-28T06:53:35.948163Z",
     "shell.execute_reply": "2022-12-28T06:53:35.946582Z",
     "shell.execute_reply.started": "2022-12-28T06:53:35.941444Z"
    }
   },
   "outputs": [],
   "source": [
    "def remove_stopwords(text):\n",
    "    new_text = []\n",
    "    \n",
    "    for word in text.split():\n",
    "        if word in stopwords.words('english'):\n",
    "            new_text.append('')\n",
    "        else:\n",
    "            new_text.append(word)\n",
    "#     x = new_text[:]\n",
    "#     new_text.clear()\n",
    "    return \" \".join(new_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:35.950199Z",
     "iopub.status.busy": "2022-12-28T06:53:35.949874Z",
     "iopub.status.idle": "2022-12-28T06:53:35.968946Z",
     "shell.execute_reply": "2022-12-28T06:53:35.967513Z",
     "shell.execute_reply.started": "2022-12-28T06:53:35.950130Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'probably  all-time favorite movie,  story  selflessness, sacrifice  dedication   noble cause,    preachy  boring.   never gets old, despite   seen   15   times'"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "remove_stopwords('probably my all-time favorite movie, a story of selflessness, sacrifice and dedication to a noble cause, but it\\'s not preachy or boring. it just never gets old, despite my having seen it some 15 or more times')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:35.971100Z",
     "iopub.status.busy": "2022-12-28T06:53:35.970750Z",
     "iopub.status.idle": "2022-12-28T06:53:35.977686Z",
     "shell.execute_reply": "2022-12-28T06:53:35.976031Z",
     "shell.execute_reply.started": "2022-12-28T06:53:35.971059Z"
    }
   },
   "outputs": [],
   "source": [
    "def remove_stopwords(text):\n",
    "    new_text = []\n",
    "    \n",
    "    for word in text.split():\n",
    "        if word in stopwords.words('english'):\n",
    "            new_text.append('')\n",
    "        else:\n",
    "            new_text.append(word)\n",
    "    x = new_text[:]\n",
    "    new_text.clear()\n",
    "    return \" \".join(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:35.980063Z",
     "iopub.status.busy": "2022-12-28T06:53:35.979167Z",
     "iopub.status.idle": "2022-12-28T06:53:35.997272Z",
     "shell.execute_reply": "2022-12-28T06:53:35.996383Z",
     "shell.execute_reply.started": "2022-12-28T06:53:35.980020Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'probably  all-time favorite movie,  story  selflessness, sacrifice  dedication   noble cause,    preachy  boring.   never gets old, despite   seen   15   times'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "remove_stopwords('probably my all-time favorite movie, a story of selflessness, sacrifice and dedication to a noble cause, but it\\'s not preachy or boring. it just never gets old, despite my having seen it some 15 or more times')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:35.999726Z",
     "iopub.status.busy": "2022-12-28T06:53:35.999205Z",
     "iopub.status.idle": "2022-12-28T06:53:36.011497Z",
     "shell.execute_reply": "2022-12-28T06:53:36.010299Z",
     "shell.execute_reply.started": "2022-12-28T06:53:35.999686Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one of the other reviewers has mentioned that ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a wonderful little production. the filming tec...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>i thought this was a wonderful way to spend ti...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>basically there's a family where a little boy ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>petter mattei's \"love in the time of money\" is...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review sentiment\n",
       "0  one of the other reviewers has mentioned that ...  positive\n",
       "1  a wonderful little production. the filming tec...  positive\n",
       "2  i thought this was a wonderful way to spend ti...  positive\n",
       "3  basically there's a family where a little boy ...  negative\n",
       "4  petter mattei's \"love in the time of money\" is...  positive"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T06:53:36.013689Z",
     "iopub.status.busy": "2022-12-28T06:53:36.013367Z",
     "iopub.status.idle": "2022-12-28T07:14:19.989709Z",
     "shell.execute_reply": "2022-12-28T07:14:19.988690Z",
     "shell.execute_reply.started": "2022-12-28T06:53:36.013647Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        one    reviewers  mentioned   watching  1 oz e...\n",
       "1         wonderful little production.  filming techniq...\n",
       "2         thought    wonderful way  spend time    hot s...\n",
       "3        basically there's  family   little boy (jake) ...\n",
       "4        petter mattei's \"love   time  money\"   visuall...\n",
       "                               ...                        \n",
       "49995     thought  movie    right good job.    creative...\n",
       "49996    bad plot, bad dialogue, bad acting, idiotic di...\n",
       "49997       catholic taught  parochial elementary schoo...\n",
       "49998    i'm going    disagree   previous comment  side...\n",
       "49999     one expects  star trek movies   high art,   f...\n",
       "Name: review, Length: 50000, dtype: object"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['review'].apply(remove_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:19.992074Z",
     "iopub.status.busy": "2022-12-28T07:14:19.991698Z",
     "iopub.status.idle": "2022-12-28T07:14:19.998443Z",
     "shell.execute_reply": "2022-12-28T07:14:19.997325Z",
     "shell.execute_reply.started": "2022-12-28T07:14:19.992031Z"
    }
   },
   "outputs": [],
   "source": [
    "import re\n",
    "def remove_emoji(text):\n",
    "    emoji_pattern = re.compile(\"[\"\n",
    "                           u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "                           u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "                           u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "                           u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "                           u\"\\U00002702-\\U000027B0\"\n",
    "                           u\"\\U000024C2-\\U0001F251\"\n",
    "                           \"]+\", flags=re.UNICODE)\n",
    "    return emoji_pattern.sub(r'', text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:19.999844Z",
     "iopub.status.busy": "2022-12-28T07:14:19.999629Z",
     "iopub.status.idle": "2022-12-28T07:14:20.017356Z",
     "shell.execute_reply": "2022-12-28T07:14:20.016198Z",
     "shell.execute_reply.started": "2022-12-28T07:14:19.999816Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Loved the movie. It was '"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "remove_emoji(\"Loved the movie. It was 😘😘\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:20.019267Z",
     "iopub.status.busy": "2022-12-28T07:14:20.019035Z",
     "iopub.status.idle": "2022-12-28T07:14:20.036457Z",
     "shell.execute_reply": "2022-12-28T07:14:20.035746Z",
     "shell.execute_reply.started": "2022-12-28T07:14:20.019237Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Lmao '"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "remove_emoji(\"Lmao 😂😂\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:20.038682Z",
     "iopub.status.busy": "2022-12-28T07:14:20.037612Z",
     "iopub.status.idle": "2022-12-28T07:14:20.278284Z",
     "shell.execute_reply": "2022-12-28T07:14:20.277068Z",
     "shell.execute_reply.started": "2022-12-28T07:14:20.038623Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python is :fire:\n"
     ]
    }
   ],
   "source": [
    "import emoji\n",
    "print(emoji.demojize('Python is 🔥'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:20.279872Z",
     "iopub.status.busy": "2022-12-28T07:14:20.279635Z",
     "iopub.status.idle": "2022-12-28T07:14:20.284699Z",
     "shell.execute_reply": "2022-12-28T07:14:20.283763Z",
     "shell.execute_reply.started": "2022-12-28T07:14:20.279842Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loved the movie. It was :face_blowing_a_kiss:\n"
     ]
    }
   ],
   "source": [
    "print(emoji.demojize('Loved the movie. It was 😘'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Using the split function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:20.285909Z",
     "iopub.status.busy": "2022-12-28T07:14:20.285680Z",
     "iopub.status.idle": "2022-12-28T07:14:20.299867Z",
     "shell.execute_reply": "2022-12-28T07:14:20.298720Z",
     "shell.execute_reply.started": "2022-12-28T07:14:20.285880Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I', 'am', 'going', 'to', 'delhi']"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# word tokenization\n",
    "sent1 = 'I am going to delhi'\n",
    "sent1.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:20.300954Z",
     "iopub.status.busy": "2022-12-28T07:14:20.300755Z",
     "iopub.status.idle": "2022-12-28T07:14:20.312316Z",
     "shell.execute_reply": "2022-12-28T07:14:20.311275Z",
     "shell.execute_reply.started": "2022-12-28T07:14:20.300929Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I am going to delhi',\n",
       " ' I will stay there for 3 days',\n",
       " \" Let's hope the trip to be great\"]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sentence tokenization\n",
    "sent2 = 'I am going to delhi. I will stay there for 3 days. Let\\'s hope the trip to be great'\n",
    "sent2.split('.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:20.314418Z",
     "iopub.status.busy": "2022-12-28T07:14:20.313801Z",
     "iopub.status.idle": "2022-12-28T07:14:20.324038Z",
     "shell.execute_reply": "2022-12-28T07:14:20.323032Z",
     "shell.execute_reply.started": "2022-12-28T07:14:20.314378Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I', 'am', 'going', 'to', 'delhi!']"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Problems with split function\n",
    "sent3 = 'I am going to delhi!'\n",
    "sent3.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:20.326101Z",
     "iopub.status.busy": "2022-12-28T07:14:20.325766Z",
     "iopub.status.idle": "2022-12-28T07:14:20.335573Z",
     "shell.execute_reply": "2022-12-28T07:14:20.334863Z",
     "shell.execute_reply.started": "2022-12-28T07:14:20.326061Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Where do think I should go? I have 3 day holiday']"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent4 = 'Where do think I should go? I have 3 day holiday'\n",
    "sent4.split('.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Regular Expression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:20.336621Z",
     "iopub.status.busy": "2022-12-28T07:14:20.336436Z",
     "iopub.status.idle": "2022-12-28T07:14:20.347938Z",
     "shell.execute_reply": "2022-12-28T07:14:20.347151Z",
     "shell.execute_reply.started": "2022-12-28T07:14:20.336597Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I', 'am', 'going', 'to', 'delhi']"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "sent3 = 'I am going to delhi!'\n",
    "tokens = re.findall(\"[\\w']+\", sent3)\n",
    "tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:20.349083Z",
     "iopub.status.busy": "2022-12-28T07:14:20.348882Z",
     "iopub.status.idle": "2022-12-28T07:14:20.361643Z",
     "shell.execute_reply": "2022-12-28T07:14:20.360683Z",
     "shell.execute_reply.started": "2022-12-28T07:14:20.349056Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Lorem Ipsum is simply dummy text of the printing and typesetting industry',\n",
       " \"\\nLorem Ipsum has been the industry's standard dummy text ever since the 1500s, \\nwhen an unknown printer took a galley of type and scrambled it to make a type specimen book.\"]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "text = \"\"\"Lorem Ipsum is simply dummy text of the printing and typesetting industry? \n",
    "Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, \n",
    "when an unknown printer took a galley of type and scrambled it to make a type specimen book.\"\"\"\n",
    "sentences = re.compile('[.!?] ').split(text)\n",
    "sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. NLTK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:20.363046Z",
     "iopub.status.busy": "2022-12-28T07:14:20.362844Z",
     "iopub.status.idle": "2022-12-28T07:14:20.373073Z",
     "shell.execute_reply": "2022-12-28T07:14:20.372408Z",
     "shell.execute_reply.started": "2022-12-28T07:14:20.363020Z"
    }
   },
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize,sent_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:20.374417Z",
     "iopub.status.busy": "2022-12-28T07:14:20.374182Z",
     "iopub.status.idle": "2022-12-28T07:14:20.399187Z",
     "shell.execute_reply": "2022-12-28T07:14:20.398310Z",
     "shell.execute_reply.started": "2022-12-28T07:14:20.374389Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I', 'am', 'going', 'to', 'visit', 'delhi', '!']"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent1 = 'I am going to visit delhi!'\n",
    "word_tokenize(sent1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:20.400929Z",
     "iopub.status.busy": "2022-12-28T07:14:20.400713Z",
     "iopub.status.idle": "2022-12-28T07:14:20.408173Z",
     "shell.execute_reply": "2022-12-28T07:14:20.407347Z",
     "shell.execute_reply.started": "2022-12-28T07:14:20.400902Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Lorem Ipsum is simply dummy text of the printing and typesetting industry?',\n",
       " \"Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, \\nwhen an unknown printer took a galley of type and scrambled it to make a type specimen book.\"]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"\"\"Lorem Ipsum is simply dummy text of the printing and typesetting industry? \n",
    "Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, \n",
    "when an unknown printer took a galley of type and scrambled it to make a type specimen book.\"\"\"\n",
    "\n",
    "sent_tokenize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:20.412137Z",
     "iopub.status.busy": "2022-12-28T07:14:20.411911Z",
     "iopub.status.idle": "2022-12-28T07:14:20.423750Z",
     "shell.execute_reply": "2022-12-28T07:14:20.422163Z",
     "shell.execute_reply.started": "2022-12-28T07:14:20.412109Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I', 'have', 'a', 'Ph.D', 'in', 'A.I']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent5 = 'I have a Ph.D in A.I'\n",
    "sent6 = \"We're here to help! mail us at nks@gmail.com\"\n",
    "sent7 = 'A 5km ride cost $10.50'\n",
    "\n",
    "word_tokenize(sent5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:20.426191Z",
     "iopub.status.busy": "2022-12-28T07:14:20.425603Z",
     "iopub.status.idle": "2022-12-28T07:14:20.437775Z",
     "shell.execute_reply": "2022-12-28T07:14:20.436727Z",
     "shell.execute_reply.started": "2022-12-28T07:14:20.426149Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['We',\n",
       " \"'re\",\n",
       " 'here',\n",
       " 'to',\n",
       " 'help',\n",
       " '!',\n",
       " 'mail',\n",
       " 'us',\n",
       " 'at',\n",
       " 'nks',\n",
       " '@',\n",
       " 'gmail.com']"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_tokenize(sent6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:20.439419Z",
     "iopub.status.busy": "2022-12-28T07:14:20.439162Z",
     "iopub.status.idle": "2022-12-28T07:14:20.451270Z",
     "shell.execute_reply": "2022-12-28T07:14:20.450681Z",
     "shell.execute_reply.started": "2022-12-28T07:14:20.439385Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['A', '5km', 'ride', 'cost', '$', '10.50']"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_tokenize(sent7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:20.452684Z",
     "iopub.status.busy": "2022-12-28T07:14:20.452487Z",
     "iopub.status.idle": "2022-12-28T07:14:32.428776Z",
     "shell.execute_reply": "2022-12-28T07:14:32.427898Z",
     "shell.execute_reply.started": "2022-12-28T07:14:20.452659Z"
    }
   },
   "outputs": [],
   "source": [
    "import spacy\n",
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:32.430260Z",
     "iopub.status.busy": "2022-12-28T07:14:32.430014Z",
     "iopub.status.idle": "2022-12-28T07:14:32.476068Z",
     "shell.execute_reply": "2022-12-28T07:14:32.474842Z",
     "shell.execute_reply.started": "2022-12-28T07:14:32.430211Z"
    }
   },
   "outputs": [],
   "source": [
    "doc1 = nlp(sent5)\n",
    "doc2 = nlp(sent6)\n",
    "doc3 = nlp(sent7)\n",
    "doc4 = nlp(sent1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:32.477645Z",
     "iopub.status.busy": "2022-12-28T07:14:32.477370Z",
     "iopub.status.idle": "2022-12-28T07:14:32.484168Z",
     "shell.execute_reply": "2022-12-28T07:14:32.483612Z",
     "shell.execute_reply.started": "2022-12-28T07:14:32.477609Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I\n",
      "am\n",
      "going\n",
      "to\n",
      "visit\n",
      "delhi\n",
      "!\n"
     ]
    }
   ],
   "source": [
    "for token in doc4:\n",
    "    print(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:32.485885Z",
     "iopub.status.busy": "2022-12-28T07:14:32.485264Z",
     "iopub.status.idle": "2022-12-28T07:14:32.500458Z",
     "shell.execute_reply": "2022-12-28T07:14:32.499546Z",
     "shell.execute_reply.started": "2022-12-28T07:14:32.485844Z"
    }
   },
   "outputs": [],
   "source": [
    "from nltk.stem.porter import PorterStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:32.502194Z",
     "iopub.status.busy": "2022-12-28T07:14:32.501824Z",
     "iopub.status.idle": "2022-12-28T07:14:32.515182Z",
     "shell.execute_reply": "2022-12-28T07:14:32.513626Z",
     "shell.execute_reply.started": "2022-12-28T07:14:32.502162Z"
    }
   },
   "outputs": [],
   "source": [
    "ps = PorterStemmer()\n",
    "def stem_words(text):\n",
    "    return \" \".join([ps.stem(word) for word in text.split()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:32.518343Z",
     "iopub.status.busy": "2022-12-28T07:14:32.517320Z",
     "iopub.status.idle": "2022-12-28T07:14:32.531078Z",
     "shell.execute_reply": "2022-12-28T07:14:32.530174Z",
     "shell.execute_reply.started": "2022-12-28T07:14:32.518282Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'walk walk walk walk'"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample = \"walk walks walking walked\"\n",
    "stem_words(sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:32.532796Z",
     "iopub.status.busy": "2022-12-28T07:14:32.532292Z",
     "iopub.status.idle": "2022-12-28T07:14:32.541668Z",
     "shell.execute_reply": "2022-12-28T07:14:32.540562Z",
     "shell.execute_reply.started": "2022-12-28T07:14:32.532763Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "probably my alltime favorite movie a story of selflessness sacrifice and dedication to a noble cause but its not preachy or boring it just never gets old despite my having seen it some 15 or more times in the last 25 years paul lukas performance brings tears to my eyes and bette davis in one of her very few truly sympathetic roles is a delight the kids are as grandma says more like dressedup midgets than children but that only makes them more fun to watch and the mothers slow awakening to whats happening in the world and under her own roof is believable and startling if i had a dozen thumbs theyd all be up for this movie\n"
     ]
    }
   ],
   "source": [
    "text = 'probably my alltime favorite movie a story of selflessness sacrifice and dedication to a noble cause but its not preachy or boring it just never gets old despite my having seen it some 15 or more times in the last 25 years paul lukas performance brings tears to my eyes and bette davis in one of her very few truly sympathetic roles is a delight the kids are as grandma says more like dressedup midgets than children but that only makes them more fun to watch and the mothers slow awakening to whats happening in the world and under her own roof is believable and startling if i had a dozen thumbs theyd all be up for this movie'\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:32.543668Z",
     "iopub.status.busy": "2022-12-28T07:14:32.543446Z",
     "iopub.status.idle": "2022-12-28T07:14:32.562812Z",
     "shell.execute_reply": "2022-12-28T07:14:32.561576Z",
     "shell.execute_reply.started": "2022-12-28T07:14:32.543638Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'probabl my alltim favorit movi a stori of selfless sacrific and dedic to a nobl caus but it not preachi or bore it just never get old despit my have seen it some 15 or more time in the last 25 year paul luka perform bring tear to my eye and bett davi in one of her veri few truli sympathet role is a delight the kid are as grandma say more like dressedup midget than children but that onli make them more fun to watch and the mother slow awaken to what happen in the world and under her own roof is believ and startl if i had a dozen thumb theyd all be up for thi movi'"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stem_words(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-28T07:14:32.565018Z",
     "iopub.status.busy": "2022-12-28T07:14:32.564317Z",
     "iopub.status.idle": "2022-12-28T07:14:34.473601Z",
     "shell.execute_reply": "2022-12-28T07:14:34.472532Z",
     "shell.execute_reply.started": "2022-12-28T07:14:32.564981Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word                Lemma               \n",
      "He                  He                  \n",
      "was                 be                  \n",
      "running             run                 \n",
      "and                 and                 \n",
      "eating              eat                 \n",
      "at                  at                  \n",
      "same                same                \n",
      "time                time                \n",
      "He                  He                  \n",
      "has                 have                \n",
      "bad                 bad                 \n",
      "habit               habit               \n",
      "of                  of                  \n",
      "swimming            swim                \n",
      "after               after               \n",
      "playing             play                \n",
      "long                long                \n",
      "hours               hours               \n",
      "in                  in                  \n",
      "the                 the                 \n",
      "Sun                 Sun                 \n"
     ]
    }
   ],
   "source": [
    "#alternative of PorterStemmer which is used some problem occurs by PorterStemmer like spelling mistake\n",
    "import nltk\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "wordnet_lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "sentence = \"He was running and eating at same time. He has bad habit of swimming after playing long hours in the Sun.\"\n",
    "punctuations=\"?:!.,;\"\n",
    "sentence_words = nltk.word_tokenize(sentence)\n",
    "for word in sentence_words:\n",
    "    if word in punctuations:\n",
    "        sentence_words.remove(word)\n",
    "\n",
    "sentence_words\n",
    "print(\"{0:20}{1:20}\".format(\"Word\",\"Lemma\"))\n",
    "for word in sentence_words:\n",
    "    print (\"{0:20}{1:20}\".format(word,wordnet_lemmatizer.lemmatize(word,pos='v')))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}