NLP_mayur.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### NLP Processing for our unstruvtured data\n",
    "Now we have our four features but by observing above table we have some string data available and as we know that machines can not understand these strings so we have transform these string respresented data in to numerical form.\n",
    "\n",
    "Here we can apply one hot encode to the subreddit but applying to title will not give proper result so for the title, we do nlp on that and trying to get one global matrix for title.\n",
    "\n",
    "To do NLP on title, we have to follow the particular pipeline to get the major words from the title. We are applying pipeline given as below.\n",
    "\n",
    "- REMOVE PUNCTUATION\n",
    "- TOKENIZATION\n",
    "- REMOVING STOPWORDS\n",
    "- STEMMING (IF NEEDED TO IMPROVE ACCURACY)\n",
    "- VECTORIZING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(df):\n",
    "    \n",
    "    df = df[df.columns.drop(list(df.filter(regex='^Cat')))]\n",
    "    df = df[df['Date'] != '27/06/2001']  #removing the date\n",
    "    df[(df['Subject'] != 'RE:') & (df['Subject'] != 'FW:') & (df['Subject'] != 'Re:')]  #removing the max same subjects\n",
    "    del df['Unnamed: 0']\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/IPython/core/interactiveshell.py:3044: DtypeWarning: Columns (9) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "DATA = 'data' #https://data.world/brianray/enron-email-dataset\n",
    "\n",
    "FILENAMES = [os.path.join(DATA, filename) for filename in os.listdir(DATA)]\n",
    "df = pd.read_csv(FILENAMES[0])\n",
    "df = preprocess(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Message-ID</th>\n",
       "      <th>Date</th>\n",
       "      <th>From</th>\n",
       "      <th>To</th>\n",
       "      <th>Subject</th>\n",
       "      <th>X-From</th>\n",
       "      <th>X-To</th>\n",
       "      <th>X-cc</th>\n",
       "      <th>X-bcc</th>\n",
       "      <th>X-Folder</th>\n",
       "      <th>X-Origin</th>\n",
       "      <th>X-FileName</th>\n",
       "      <th>content</th>\n",
       "      <th>user</th>\n",
       "      <th>labeled</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;18782981.1075855378110.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>2001-05-14 23:39:00</td>\n",
       "      <td>frozenset({'phillip.allen@enron.com'})</td>\n",
       "      <td>frozenset({'tim.belden@enron.com'})</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Tim Belden &lt;Tim Belden/Enron@EnronXGate&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>\\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...</td>\n",
       "      <td>Allen-P</td>\n",
       "      <td>pallen (Non-Privileged).pst</td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>allen-p</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>2001-05-04 20:51:00</td>\n",
       "      <td>frozenset({'phillip.allen@enron.com'})</td>\n",
       "      <td>frozenset({'john.lavorato@enron.com'})</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>John J Lavorato &lt;John J Lavorato/ENRON@enronXg...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>\\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...</td>\n",
       "      <td>Allen-P</td>\n",
       "      <td>pallen (Non-Privileged).pst</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>allen-p</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>2000-10-18 10:00:00</td>\n",
       "      <td>frozenset({'phillip.allen@enron.com'})</td>\n",
       "      <td>frozenset({'leah.arsdall@enron.com'})</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Leah Van Arsdall</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Allen-P</td>\n",
       "      <td>pallen.nsf</td>\n",
       "      <td>test successful. way to go!!!</td>\n",
       "      <td>allen-p</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;13505866.1075863688222.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>2000-10-23 13:13:00</td>\n",
       "      <td>frozenset({'phillip.allen@enron.com'})</td>\n",
       "      <td>frozenset({'randall.gay@enron.com'})</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Randall L Gay</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Allen-P</td>\n",
       "      <td>pallen.nsf</td>\n",
       "      <td>Randy, Can you send me a schedule of the salar...</td>\n",
       "      <td>allen-p</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>2000-08-31 12:07:00</td>\n",
       "      <td>frozenset({'phillip.allen@enron.com'})</td>\n",
       "      <td>frozenset({'greg.piper@enron.com'})</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Greg Piper</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Allen-P</td>\n",
       "      <td>pallen.nsf</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>allen-p</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Message-ID                 Date  \\\n",
       "0  <18782981.1075855378110.JavaMail.evans@thyme>  2001-05-14 23:39:00   \n",
       "1  <15464986.1075855378456.JavaMail.evans@thyme>  2001-05-04 20:51:00   \n",
       "2  <24216240.1075855687451.JavaMail.evans@thyme>  2000-10-18 10:00:00   \n",
       "3  <13505866.1075863688222.JavaMail.evans@thyme>  2000-10-23 13:13:00   \n",
       "4  <30922949.1075863688243.JavaMail.evans@thyme>  2000-08-31 12:07:00   \n",
       "\n",
       "                                     From  \\\n",
       "0  frozenset({'phillip.allen@enron.com'})   \n",
       "1  frozenset({'phillip.allen@enron.com'})   \n",
       "2  frozenset({'phillip.allen@enron.com'})   \n",
       "3  frozenset({'phillip.allen@enron.com'})   \n",
       "4  frozenset({'phillip.allen@enron.com'})   \n",
       "\n",
       "                                       To    Subject           X-From  \\\n",
       "0     frozenset({'tim.belden@enron.com'})        NaN  Phillip K Allen   \n",
       "1  frozenset({'john.lavorato@enron.com'})        Re:  Phillip K Allen   \n",
       "2   frozenset({'leah.arsdall@enron.com'})   Re: test  Phillip K Allen   \n",
       "3    frozenset({'randall.gay@enron.com'})        NaN  Phillip K Allen   \n",
       "4     frozenset({'greg.piper@enron.com'})  Re: Hello  Phillip K Allen   \n",
       "\n",
       "                                                X-To X-cc X-bcc  \\\n",
       "0           Tim Belden <Tim Belden/Enron@EnronXGate>  NaN   NaN   \n",
       "1  John J Lavorato <John J Lavorato/ENRON@enronXg...  NaN   NaN   \n",
       "2                                   Leah Van Arsdall  NaN   NaN   \n",
       "3                                      Randall L Gay  NaN   NaN   \n",
       "4                                         Greg Piper  NaN   NaN   \n",
       "\n",
       "                                            X-Folder X-Origin  \\\n",
       "0  \\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...  Allen-P   \n",
       "1  \\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...  Allen-P   \n",
       "2    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Allen-P   \n",
       "3    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Allen-P   \n",
       "4    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Allen-P   \n",
       "\n",
       "                    X-FileName  \\\n",
       "0  pallen (Non-Privileged).pst   \n",
       "1  pallen (Non-Privileged).pst   \n",
       "2                   pallen.nsf   \n",
       "3                   pallen.nsf   \n",
       "4                   pallen.nsf   \n",
       "\n",
       "                                             content     user  labeled  \n",
       "0                               Here is our forecast  allen-p    False  \n",
       "1  Traveling to have a business meeting takes the...  allen-p    False  \n",
       "2                      test successful. way to go!!!  allen-p    False  \n",
       "3  Randy, Can you send me a schedule of the salar...  allen-p    False  \n",
       "4                  Let's shoot for Tuesday at 11:45.  allen-p    False  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfEmail = df[['Subject', 'content']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Here is our forecast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful. way to go!!!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Randy, Can you send me a schedule of the salar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Subject                                            content\n",
       "0        NaN                               Here is our forecast\n",
       "1        Re:  Traveling to have a business meeting takes the...\n",
       "2   Re: test                      test successful. way to go!!!\n",
       "3        NaN  Randy, Can you send me a schedule of the salar...\n",
       "4  Re: Hello                  Let's shoot for Tuesday at 11:45."
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfEmail.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Remove Punctuation\n",
    "In this method, We are removing all the following the punctuations from content and subject"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import string\n",
    "\n",
    "print(string.punctuation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  import sys\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject</th>\n",
       "      <th>content</th>\n",
       "      <th>content_removed_punctuation</th>\n",
       "      <th>Subject_removed_punctuation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>here is our forecast</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>traveling to have a business meeting takes the...</td>\n",
       "      <td>re</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful. way to go!!!</td>\n",
       "      <td>test successful way to go</td>\n",
       "      <td>re test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Randy, Can you send me a schedule of the salar...</td>\n",
       "      <td>randy can you send me a schedule of the salary...</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>lets shoot for tuesday at 1145</td>\n",
       "      <td>re hello</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Subject                                            content  \\\n",
       "0        NaN                               Here is our forecast   \n",
       "1        Re:  Traveling to have a business meeting takes the...   \n",
       "2   Re: test                      test successful. way to go!!!   \n",
       "3        NaN  Randy, Can you send me a schedule of the salar...   \n",
       "4  Re: Hello                  Let's shoot for Tuesday at 11:45.   \n",
       "\n",
       "                         content_removed_punctuation  \\\n",
       "0                               here is our forecast   \n",
       "1  traveling to have a business meeting takes the...   \n",
       "2                          test successful way to go   \n",
       "3  randy can you send me a schedule of the salary...   \n",
       "4                     lets shoot for tuesday at 1145   \n",
       "\n",
       "  Subject_removed_punctuation  \n",
       "0                         nan  \n",
       "1                          re  \n",
       "2                     re test  \n",
       "3                         nan  \n",
       "4                    re hello  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def remove_punc(text):\n",
    "    text_nopunct = \"\".join([char for char in text if char not in string.punctuation])\n",
    "    return text_nopunct\n",
    "\n",
    "''' python is case sensetive for that A and a is diffrent thats why lower()'''\n",
    "dfEmail['content_removed_punctuation'] = dfEmail['content'].apply(lambda x: remove_punc(str(x).lower()))\n",
    "dfEmail['Subject_removed_punctuation'] = dfEmail['Subject'].apply(lambda x: remove_punc(str(x).lower()))\n",
    "dfEmail.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see the difference in text by navigationg the columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Tokenization\n",
    "Tokenization is the act of breaking up a sequence of strings into pieces such as words, keywords, phrases, symbols and other elements called tokens. Tokens can be individual words, phrases or even whole sentences. In the process of tokenization, some characters like punctuation marks are discarded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:10: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Remove the CWD from sys.path while we load stuff.\n",
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:11: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # This is added back by InteractiveShellApp.init_path()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject</th>\n",
       "      <th>content</th>\n",
       "      <th>Subject_tokenize</th>\n",
       "      <th>content_tokenize</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[here, is, our, forecast]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>[re]</td>\n",
       "      <td>[traveling, to, have, a, business, meeting, ta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful. way to go!!!</td>\n",
       "      <td>[re, test]</td>\n",
       "      <td>[test, successful, way, to, go]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Randy, Can you send me a schedule of the salar...</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[randy, can, you, send, me, a, schedule, of, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>[re, hello]</td>\n",
       "      <td>[lets, shoot, for, tuesday, at, 1145]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Subject                                            content  \\\n",
       "0        NaN                               Here is our forecast   \n",
       "1        Re:  Traveling to have a business meeting takes the...   \n",
       "2   Re: test                      test successful. way to go!!!   \n",
       "3        NaN  Randy, Can you send me a schedule of the salar...   \n",
       "4  Re: Hello                  Let's shoot for Tuesday at 11:45.   \n",
       "\n",
       "  Subject_tokenize                                   content_tokenize  \n",
       "0            [nan]                          [here, is, our, forecast]  \n",
       "1             [re]  [traveling, to, have, a, business, meeting, ta...  \n",
       "2       [re, test]                    [test, successful, way, to, go]  \n",
       "3            [nan]  [randy, can, you, send, me, a, schedule, of, t...  \n",
       "4      [re, hello]              [lets, shoot, for, tuesday, at, 1145]  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "\n",
    "def tokenize(text):\n",
    "    # Split word non word\n",
    "    tokens = re.split('\\W+', text)\n",
    "    return tokens\n",
    "\n",
    "##EACH ROW\n",
    "dfEmail['Subject_tokenize'] = dfEmail['Subject_removed_punctuation'].apply(lambda x: tokenize(x))\n",
    "dfEmail['content_tokenize'] = dfEmail['content_removed_punctuation'].apply(lambda x: tokenize(x))\n",
    "del dfEmail['content_removed_punctuation']\n",
    "del dfEmail['Subject_removed_punctuation']\n",
    "dfEmail.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that each words in the sentence a"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Remove StepWords\n",
    "StepWords are those which are words which does not play importance in sentences. Like I am going to watch movie. where 'I', 'am' and 'to' plays part of the stopwords.\n",
    "\n",
    "Here we will use nltk package to remvoe stopwords."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NLTK (DOWNLOAD ALL PACKAGES TO PERFORM NLP OPERATION)\n",
      "UNCOMMENT FOLLOWING LINE To GET NLTK DOWNLOADED\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "print('NLTK (DOWNLOAD ALL PACKAGES TO PERFORM NLP OPERATION)')\n",
    "\n",
    "print('UNCOMMENT FOLLOWING LINE To GET NLTK DOWNLOADED')\n",
    "#nltk.download()\n",
    "stopword = nltk.corpus.stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"\n",
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject</th>\n",
       "      <th>content</th>\n",
       "      <th>Subject_nostopwords</th>\n",
       "      <th>content_nostopwords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[forecast]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[traveling, business, meeting, takes, fun, tri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful. way to go!!!</td>\n",
       "      <td>[test]</td>\n",
       "      <td>[test, successful, way, go]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Randy, Can you send me a schedule of the salar...</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[randy, send, schedule, salary, level, everyon...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>[hello]</td>\n",
       "      <td>[lets, shoot, tuesday, 1145]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Subject                                            content  \\\n",
       "0        NaN                               Here is our forecast   \n",
       "1        Re:  Traveling to have a business meeting takes the...   \n",
       "2   Re: test                      test successful. way to go!!!   \n",
       "3        NaN  Randy, Can you send me a schedule of the salar...   \n",
       "4  Re: Hello                  Let's shoot for Tuesday at 11:45.   \n",
       "\n",
       "  Subject_nostopwords                                content_nostopwords  \n",
       "0               [nan]                                         [forecast]  \n",
       "1                  []  [traveling, business, meeting, takes, fun, tri...  \n",
       "2              [test]                        [test, successful, way, go]  \n",
       "3               [nan]  [randy, send, schedule, salary, level, everyon...  \n",
       "4             [hello]                       [lets, shoot, tuesday, 1145]  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def remove_stopwords(tokenized_list):\n",
    "    text = [word for word in tokenized_list if word not in stopword]\n",
    "    return text\n",
    "\n",
    "dfEmail['Subject_nostopwords'] = dfEmail['Subject_tokenize'].apply(lambda x: remove_stopwords(x))\n",
    "dfEmail['content_nostopwords'] = dfEmail['content_tokenize'].apply(lambda x: remove_stopwords(x))\n",
    "del dfEmail['Subject_tokenize']\n",
    "del dfEmail['content_tokenize']\n",
    "dfEmail.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Stemming & Lemmatization\n",
    "\n",
    "Stemming and Lemmatization are Text Normalization (or sometimes called Word Normalization) techniques in the field of Natural Language Processing that are used to prepare text, words, and documents for further processing. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### Stemming Algorithms and Code\n",
    "\n",
    " different stemmers available in different languages in Python nltk. For the English language, you can choose between <b>PorterStammer</b> or <b>LancasterStammer</b>, PorterStemmer being the oldest one originally developed in 1979. LancasterStemmer was developed in 1990 and uses a more aggressive approach than Porter Stemming Algorithm."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### PorterStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Porter Stemmer\n",
      "cat\n",
      "troubl\n",
      "troubl\n",
      "troubl\n",
      "Lancaster Stemmer\n",
      "cat\n",
      "troubl\n",
      "troubl\n",
      "troubl\n"
     ]
    }
   ],
   "source": [
    "from nltk.stem import PorterStemmer\n",
    "from nltk.stem import LancasterStemmer\n",
    "\n",
    "#create an object of class PorterStemmer\n",
    "porter = PorterStemmer()\n",
    "lancaster=LancasterStemmer()\n",
    "#proide a word to be stemmed\n",
    "print(\"Porter Stemmer\")\n",
    "print(porter.stem(\"cats\"))\n",
    "print(porter.stem(\"trouble\"))\n",
    "print(porter.stem(\"troubling\"))\n",
    "print(porter.stem(\"troubled\"))\n",
    "print(\"Lancaster Stemmer\")\n",
    "print(lancaster.stem(\"cats\"))\n",
    "print(lancaster.stem(\"trouble\"))\n",
    "print(lancaster.stem(\"troubling\"))\n",
    "print(lancaster.stem(\"troubled\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"\n",
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject</th>\n",
       "      <th>content</th>\n",
       "      <th>Subject_nostopwords</th>\n",
       "      <th>content_nostopwords</th>\n",
       "      <th>Subject_port_stem</th>\n",
       "      <th>content_port_stem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[forecast]</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[forecast]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[traveling, business, meeting, takes, fun, tri...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[travel, busi, meet, take, fun, trip, especi, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful. way to go!!!</td>\n",
       "      <td>[test]</td>\n",
       "      <td>[test, successful, way, go]</td>\n",
       "      <td>[test]</td>\n",
       "      <td>[test, success, way, go]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Randy, Can you send me a schedule of the salar...</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[randy, send, schedule, salary, level, everyon...</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[randi, send, schedul, salari, level, everyon,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>[hello]</td>\n",
       "      <td>[lets, shoot, tuesday, 1145]</td>\n",
       "      <td>[hello]</td>\n",
       "      <td>[let, shoot, tuesday, 1145]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Subject                                            content  \\\n",
       "0        NaN                               Here is our forecast   \n",
       "1        Re:  Traveling to have a business meeting takes the...   \n",
       "2   Re: test                      test successful. way to go!!!   \n",
       "3        NaN  Randy, Can you send me a schedule of the salar...   \n",
       "4  Re: Hello                  Let's shoot for Tuesday at 11:45.   \n",
       "\n",
       "  Subject_nostopwords                                content_nostopwords  \\\n",
       "0               [nan]                                         [forecast]   \n",
       "1                  []  [traveling, business, meeting, takes, fun, tri...   \n",
       "2              [test]                        [test, successful, way, go]   \n",
       "3               [nan]  [randy, send, schedule, salary, level, everyon...   \n",
       "4             [hello]                       [lets, shoot, tuesday, 1145]   \n",
       "\n",
       "  Subject_port_stem                                  content_port_stem  \n",
       "0             [nan]                                         [forecast]  \n",
       "1                []  [travel, busi, meet, take, fun, trip, especi, ...  \n",
       "2            [test]                           [test, success, way, go]  \n",
       "3             [nan]  [randi, send, schedul, salari, level, everyon,...  \n",
       "4           [hello]                        [let, shoot, tuesday, 1145]  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def port_stem(tokenized_list):\n",
    "    text = [porter.stem(word) for word in tokenized_list]\n",
    "    return text\n",
    "\n",
    "dfEmail['Subject_port_stem'] = dfEmail['Subject_nostopwords'].apply(lambda x: port_stem(x))\n",
    "dfEmail['content_port_stem'] = dfEmail['content_nostopwords'].apply(lambda x: port_stem(x))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject_nostopwords</th>\n",
       "      <th>Subject_port_stem</th>\n",
       "      <th>content_nostopwords</th>\n",
       "      <th>content_port_stem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[nan]</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[forecast]</td>\n",
       "      <td>[forecast]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[traveling, business, meeting, takes, fun, tri...</td>\n",
       "      <td>[travel, busi, meet, take, fun, trip, especi, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[test]</td>\n",
       "      <td>[test]</td>\n",
       "      <td>[test, successful, way, go]</td>\n",
       "      <td>[test, success, way, go]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[nan]</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[randy, send, schedule, salary, level, everyon...</td>\n",
       "      <td>[randi, send, schedul, salari, level, everyon,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[hello]</td>\n",
       "      <td>[hello]</td>\n",
       "      <td>[lets, shoot, tuesday, 1145]</td>\n",
       "      <td>[let, shoot, tuesday, 1145]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[hello]</td>\n",
       "      <td>[hello]</td>\n",
       "      <td>[greg, either, next, tuesday, thursday, phillip]</td>\n",
       "      <td>[greg, either, next, tuesday, thursday, phillip]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[nan]</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[please, cc, following, distribution, list, up...</td>\n",
       "      <td>[pleas, cc, follow, distribut, list, updat, ph...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>[prc, review, phone, calls]</td>\n",
       "      <td>[prc, review, phone, call]</td>\n",
       "      <td>[morning, 10, 1130]</td>\n",
       "      <td>[morn, 10, 1130]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>[high, speed, internet, access]</td>\n",
       "      <td>[high, speed, internet, access]</td>\n",
       "      <td>[1, login, pallen, pw, ke9davis, dont, think, ...</td>\n",
       "      <td>[1, login, pallen, pw, ke9davi, dont, think, r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>[fw, fixed, forward, collar, floor, gas, price...</td>\n",
       "      <td>[fw, fix, forward, collar, floor, ga, price, t...</td>\n",
       "      <td>[, forwarded, phillip, k, allenhouect, 1016200...</td>\n",
       "      <td>[, forward, phillip, k, allenhouect, 10162000,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 Subject_nostopwords  \\\n",
       "0                                              [nan]   \n",
       "1                                                 []   \n",
       "2                                             [test]   \n",
       "3                                              [nan]   \n",
       "4                                            [hello]   \n",
       "5                                            [hello]   \n",
       "6                                              [nan]   \n",
       "7                        [prc, review, phone, calls]   \n",
       "8                    [high, speed, internet, access]   \n",
       "9  [fw, fixed, forward, collar, floor, gas, price...   \n",
       "\n",
       "                                   Subject_port_stem  \\\n",
       "0                                              [nan]   \n",
       "1                                                 []   \n",
       "2                                             [test]   \n",
       "3                                              [nan]   \n",
       "4                                            [hello]   \n",
       "5                                            [hello]   \n",
       "6                                              [nan]   \n",
       "7                         [prc, review, phone, call]   \n",
       "8                    [high, speed, internet, access]   \n",
       "9  [fw, fix, forward, collar, floor, ga, price, t...   \n",
       "\n",
       "                                 content_nostopwords  \\\n",
       "0                                         [forecast]   \n",
       "1  [traveling, business, meeting, takes, fun, tri...   \n",
       "2                        [test, successful, way, go]   \n",
       "3  [randy, send, schedule, salary, level, everyon...   \n",
       "4                       [lets, shoot, tuesday, 1145]   \n",
       "5   [greg, either, next, tuesday, thursday, phillip]   \n",
       "6  [please, cc, following, distribution, list, up...   \n",
       "7                                [morning, 10, 1130]   \n",
       "8  [1, login, pallen, pw, ke9davis, dont, think, ...   \n",
       "9  [, forwarded, phillip, k, allenhouect, 1016200...   \n",
       "\n",
       "                                   content_port_stem  \n",
       "0                                         [forecast]  \n",
       "1  [travel, busi, meet, take, fun, trip, especi, ...  \n",
       "2                           [test, success, way, go]  \n",
       "3  [randi, send, schedul, salari, level, everyon,...  \n",
       "4                        [let, shoot, tuesday, 1145]  \n",
       "5   [greg, either, next, tuesday, thursday, phillip]  \n",
       "6  [pleas, cc, follow, distribut, list, updat, ph...  \n",
       "7                                   [morn, 10, 1130]  \n",
       "8  [1, login, pallen, pw, ke9davi, dont, think, r...  \n",
       "9  [, forward, phillip, k, allenhouect, 10162000,...  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfEmail[['Subject_nostopwords','Subject_port_stem', 'content_nostopwords','content_port_stem']].head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### LancasterStemmer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"\n",
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "def lancaster_stem(tokenized_list):\n",
    "    text = [lancaster.stem(word) for word in tokenized_list]\n",
    "    return text\n",
    "\n",
    "dfEmail['Subject_lancaster'] = dfEmail['Subject_nostopwords'].apply(lambda x: lancaster_stem(x))\n",
    "dfEmail['content_lancaster'] = dfEmail['content_nostopwords'].apply(lambda x: lancaster_stem(x))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject_nostopwords</th>\n",
       "      <th>Subject_lancaster</th>\n",
       "      <th>content_nostopwords</th>\n",
       "      <th>content_lancaster</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[nan]</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[forecast]</td>\n",
       "      <td>[forecast]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[traveling, business, meeting, takes, fun, tri...</td>\n",
       "      <td>[travel, busy, meet, tak, fun, trip, espec, pr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[test]</td>\n",
       "      <td>[test]</td>\n",
       "      <td>[test, successful, way, go]</td>\n",
       "      <td>[test, success, way, go]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[nan]</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[randy, send, schedule, salary, level, everyon...</td>\n",
       "      <td>[randy, send, schedule, sal, level, everyon, s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[hello]</td>\n",
       "      <td>[hello]</td>\n",
       "      <td>[lets, shoot, tuesday, 1145]</td>\n",
       "      <td>[let, shoot, tuesday, 1145]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[hello]</td>\n",
       "      <td>[hello]</td>\n",
       "      <td>[greg, either, next, tuesday, thursday, phillip]</td>\n",
       "      <td>[greg, eith, next, tuesday, thursday, phillip]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[nan]</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[please, cc, following, distribution, list, up...</td>\n",
       "      <td>[pleas, cc, follow, distribut, list, upd, phil...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>[prc, review, phone, calls]</td>\n",
       "      <td>[prc, review, phon, cal]</td>\n",
       "      <td>[morning, 10, 1130]</td>\n",
       "      <td>[morn, 10, 1130]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>[high, speed, internet, access]</td>\n",
       "      <td>[high, spee, internet, access]</td>\n",
       "      <td>[1, login, pallen, pw, ke9davis, dont, think, ...</td>\n",
       "      <td>[1, login, pal, pw, ke9davis, dont, think, req...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>[fw, fixed, forward, collar, floor, gas, price...</td>\n",
       "      <td>[fw, fix, forward, coll, flo, gas, pric, term]</td>\n",
       "      <td>[, forwarded, phillip, k, allenhouect, 1016200...</td>\n",
       "      <td>[, forward, phillip, k, allenhouect, 10162000,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 Subject_nostopwords  \\\n",
       "0                                              [nan]   \n",
       "1                                                 []   \n",
       "2                                             [test]   \n",
       "3                                              [nan]   \n",
       "4                                            [hello]   \n",
       "5                                            [hello]   \n",
       "6                                              [nan]   \n",
       "7                        [prc, review, phone, calls]   \n",
       "8                    [high, speed, internet, access]   \n",
       "9  [fw, fixed, forward, collar, floor, gas, price...   \n",
       "\n",
       "                                Subject_lancaster  \\\n",
       "0                                           [nan]   \n",
       "1                                              []   \n",
       "2                                          [test]   \n",
       "3                                           [nan]   \n",
       "4                                         [hello]   \n",
       "5                                         [hello]   \n",
       "6                                           [nan]   \n",
       "7                        [prc, review, phon, cal]   \n",
       "8                  [high, spee, internet, access]   \n",
       "9  [fw, fix, forward, coll, flo, gas, pric, term]   \n",
       "\n",
       "                                 content_nostopwords  \\\n",
       "0                                         [forecast]   \n",
       "1  [traveling, business, meeting, takes, fun, tri...   \n",
       "2                        [test, successful, way, go]   \n",
       "3  [randy, send, schedule, salary, level, everyon...   \n",
       "4                       [lets, shoot, tuesday, 1145]   \n",
       "5   [greg, either, next, tuesday, thursday, phillip]   \n",
       "6  [please, cc, following, distribution, list, up...   \n",
       "7                                [morning, 10, 1130]   \n",
       "8  [1, login, pallen, pw, ke9davis, dont, think, ...   \n",
       "9  [, forwarded, phillip, k, allenhouect, 1016200...   \n",
       "\n",
       "                                   content_lancaster  \n",
       "0                                         [forecast]  \n",
       "1  [travel, busy, meet, tak, fun, trip, espec, pr...  \n",
       "2                           [test, success, way, go]  \n",
       "3  [randy, send, schedule, sal, level, everyon, s...  \n",
       "4                        [let, shoot, tuesday, 1145]  \n",
       "5     [greg, eith, next, tuesday, thursday, phillip]  \n",
       "6  [pleas, cc, follow, distribut, list, upd, phil...  \n",
       "7                                   [morn, 10, 1130]  \n",
       "8  [1, login, pal, pw, ke9davis, dont, think, req...  \n",
       "9  [, forward, phillip, k, allenhouect, 10162000,...  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfEmail[['Subject_nostopwords','Subject_lancaster', 'content_nostopwords','content_lancaster']].head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Lemmatization \n",
    "\n",
    "Lemmatization, unlike Stemming, reduces the inflected words properly ensuring that the root word belongs to the language. In Lemmatization root word is called Lemma. A lemma (plural lemmas or lemmata) is the canonical form, dictionary form, or citation form of a set of words\n",
    "\n",
    "For example, runs, running, ran are all forms of the word run, therefore run is the lemma of all these words. Because lemmatization returns an actual word of the language, it is used where it is necessary to get valid words.\n",
    "\n",
    "Python NLTK provides WordNet Lemmatizer that uses the WordNet Database to lookup lemmas of words.\n",
    "\n",
    "<b>Note: Download the WordNet corpora from NLTK downloader before using the WordNet Lemmatizer<b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word                Lemma               \n",
      "He                  He                  \n",
      "was                 be                  \n",
      "running             run                 \n",
      "and                 and                 \n",
      "eating              eat                 \n",
      "at                  at                  \n",
      "same                same                \n",
      "time                time                \n",
      "He                  He                  \n",
      "has                 have                \n",
      "bad                 bad                 \n",
      "habit               habit               \n",
      "of                  of                  \n",
      "swimming            swim                \n",
      "after               after               \n",
      "playing             play                \n",
      "long                long                \n",
      "hours               hours               \n",
      "in                  in                  \n",
      "the                 the                 \n",
      "Sun                 Sun                 \n"
     ]
    }
   ],
   "source": [
    "from nltk.stem import WordNetLemmatizer\n",
    "wordnet_lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "sentence = \"He was running and eating at same time. He has bad habit of swimming after playing long hours in the Sun.\"\n",
    "punctuations=\"?:!.,;\"\n",
    "sentence_words = nltk.word_tokenize(sentence)\n",
    "for word in sentence_words:\n",
    "    if word in punctuations:\n",
    "        sentence_words.remove(word)\n",
    "\n",
    "sentence_words\n",
    "print(\"{0:20}{1:20}\".format(\"Word\",\"Lemma\"))\n",
    "for word in sentence_words:\n",
    "#     print (\"{0:20}{1:20}\".format(word,wordnet_lemmatizer.lemmatize(word))) this just removes plurals\n",
    "    print (\"{0:20}{1:20}\".format(word,wordnet_lemmatizer.lemmatize(word, pos=\"v\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  after removing the cwd from sys.path.\n",
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "def lemma(tokenized_list):\n",
    "    return [wordnet_lemmatizer.lemmatize(word, pos=\"v\") for word in tokenized_list]   \n",
    "\n",
    "dfEmail['Subject_lemma'] = dfEmail['Subject_nostopwords'].apply(lambda x: lemma(x))\n",
    "dfEmail['content_lemma'] = dfEmail['content_nostopwords'].apply(lambda x: lemma(x))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject_nostopwords</th>\n",
       "      <th>Subject_lemma</th>\n",
       "      <th>content_nostopwords</th>\n",
       "      <th>content_lemma</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[nan]</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[forecast]</td>\n",
       "      <td>[forecast]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[traveling, business, meeting, takes, fun, tri...</td>\n",
       "      <td>[travel, business, meet, take, fun, trip, espe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[test]</td>\n",
       "      <td>[test]</td>\n",
       "      <td>[test, successful, way, go]</td>\n",
       "      <td>[test, successful, way, go]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[nan]</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[randy, send, schedule, salary, level, everyon...</td>\n",
       "      <td>[randy, send, schedule, salary, level, everyon...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[hello]</td>\n",
       "      <td>[hello]</td>\n",
       "      <td>[lets, shoot, tuesday, 1145]</td>\n",
       "      <td>[let, shoot, tuesday, 1145]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[hello]</td>\n",
       "      <td>[hello]</td>\n",
       "      <td>[greg, either, next, tuesday, thursday, phillip]</td>\n",
       "      <td>[greg, either, next, tuesday, thursday, phillip]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[nan]</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[please, cc, following, distribution, list, up...</td>\n",
       "      <td>[please, cc, follow, distribution, list, updat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>[prc, review, phone, calls]</td>\n",
       "      <td>[prc, review, phone, call]</td>\n",
       "      <td>[morning, 10, 1130]</td>\n",
       "      <td>[morning, 10, 1130]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>[high, speed, internet, access]</td>\n",
       "      <td>[high, speed, internet, access]</td>\n",
       "      <td>[1, login, pallen, pw, ke9davis, dont, think, ...</td>\n",
       "      <td>[1, login, pallen, pw, ke9davis, dont, think, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>[fw, fixed, forward, collar, floor, gas, price...</td>\n",
       "      <td>[fw, fix, forward, collar, floor, gas, price, ...</td>\n",
       "      <td>[, forwarded, phillip, k, allenhouect, 1016200...</td>\n",
       "      <td>[, forward, phillip, k, allenhouect, 10162000,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 Subject_nostopwords  \\\n",
       "0                                              [nan]   \n",
       "1                                                 []   \n",
       "2                                             [test]   \n",
       "3                                              [nan]   \n",
       "4                                            [hello]   \n",
       "5                                            [hello]   \n",
       "6                                              [nan]   \n",
       "7                        [prc, review, phone, calls]   \n",
       "8                    [high, speed, internet, access]   \n",
       "9  [fw, fixed, forward, collar, floor, gas, price...   \n",
       "\n",
       "                                       Subject_lemma  \\\n",
       "0                                              [nan]   \n",
       "1                                                 []   \n",
       "2                                             [test]   \n",
       "3                                              [nan]   \n",
       "4                                            [hello]   \n",
       "5                                            [hello]   \n",
       "6                                              [nan]   \n",
       "7                         [prc, review, phone, call]   \n",
       "8                    [high, speed, internet, access]   \n",
       "9  [fw, fix, forward, collar, floor, gas, price, ...   \n",
       "\n",
       "                                 content_nostopwords  \\\n",
       "0                                         [forecast]   \n",
       "1  [traveling, business, meeting, takes, fun, tri...   \n",
       "2                        [test, successful, way, go]   \n",
       "3  [randy, send, schedule, salary, level, everyon...   \n",
       "4                       [lets, shoot, tuesday, 1145]   \n",
       "5   [greg, either, next, tuesday, thursday, phillip]   \n",
       "6  [please, cc, following, distribution, list, up...   \n",
       "7                                [morning, 10, 1130]   \n",
       "8  [1, login, pallen, pw, ke9davis, dont, think, ...   \n",
       "9  [, forwarded, phillip, k, allenhouect, 1016200...   \n",
       "\n",
       "                                       content_lemma  \n",
       "0                                         [forecast]  \n",
       "1  [travel, business, meet, take, fun, trip, espe...  \n",
       "2                        [test, successful, way, go]  \n",
       "3  [randy, send, schedule, salary, level, everyon...  \n",
       "4                        [let, shoot, tuesday, 1145]  \n",
       "5   [greg, either, next, tuesday, thursday, phillip]  \n",
       "6  [please, cc, follow, distribution, list, updat...  \n",
       "7                                [morning, 10, 1130]  \n",
       "8  [1, login, pallen, pw, ke9davis, dont, think, ...  \n",
       "9  [, forward, phillip, k, allenhouect, 10162000,...  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfEmail[['Subject_nostopwords','Subject_lemma', 'content_nostopwords','content_lemma']].head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now compare the above 3 steeming and lemmatization columns with the nostopwords column and analyze which technique is working best for us. It is still part of EDA but by going through the above port stemming, lancasster stemming and lemmatization columns, The lemmatization works as appropriate as it is not changing the meaning of the word and still truncating the extra characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "del dfEmail['content_lancaster']\n",
    "del dfEmail['content_port_stem']\n",
    "del dfEmail['Subject_lancaster']\n",
    "del dfEmail['Subject_port_stem']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'dfEmail' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-057e9a607fe6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdfEmail\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'dfEmail' is not defined"
     ]
    }
   ],
   "source": [
    "dfEmail.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}