hands_on_machine_learning/3/spam_classifier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise 4: Build a spam classifier.\n",
    "\n",
    "In this exercise, we will use spam assassin data to build a classifier which detects whether an email is a spam or not.\n",
    "\n",
    "**Note that, we will not use advanced NLP techniques, as goal of this notebook is to just build a classifier based on what we learned so far. We might go in-depth algorithms at a later time, when we have learned algorithms in-depth.**\n",
    "\n",
    "Let's fetch the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tarfile\n",
    "import urllib\n",
    "\n",
    "\n",
    "DOWNLOAD_ROOT = \"http://spamassassin.apache.org/old/publiccorpus/\"\n",
    "HAM_URL = DOWNLOAD_ROOT + \"20030228_easy_ham.tar.bz2\"\n",
    "SPAM_URL = DOWNLOAD_ROOT + \"20030228_spam.tar.bz2\"\n",
    "SPAM_PATH = os.path.join(\"datasets\", \"spam\")\n",
    "\n",
    "def download_spam_data(spam_url=SPAM_URL, spam_path=SPAM_PATH):\n",
    "    if not os.path.isdir(spam_path):\n",
    "        os.makedirs(spam_path)\n",
    "    for filename, url in ((\"ham.tar.bz2\", HAM_URL), (\"spam.tar.bz2\", SPAM_URL)):\n",
    "        path = os.path.join(spam_path, filename)\n",
    "        if not os.path.isfile(path):\n",
    "            urllib.request.urlretrieve(url, path)\n",
    "        tar_bz2_file = tarfile.open(path)\n",
    "        tar_bz2_file.extractall(path=SPAM_PATH)\n",
    "        tar_bz2_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "download_spam_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, let's load the emails we downloaded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "HAM_DIR = os.path.join(SPAM_PATH, \"easy_ham\")\n",
    "SPAM_DIR = os.path.join(SPAM_PATH, \"spam\")\n",
    "ham_filenames = [name for name in sorted(os.listdir(HAM_DIR)) if len(name) > 20]\n",
    "spam_filenames = [name for name in sorted(os.listdir(SPAM_DIR)) if len(name) > 20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2500"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ham_filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(spam_filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'From exmh-workers-admin@redhat.com  Thu Aug 22 12:36:23 2002\\nReturn-Path: <exmh-workers-admin@spamassassin.taint.org>\\nDelivered-To: zzzz@localhost.netnoteinc.com\\nReceived: from localhost (localhost [127.0.0.1])\\n\\tby phobos.labs.netnoteinc.com (Postfix) with ESMTP id D03E543C36\\n\\tfor <zzzz@localhost>; Thu, 22 Aug 2002 07:36:16 -0400 (EDT)\\nReceived: from phobos [127.0.0.1]\\n\\tby localhost with IMAP (fetchmail-5.9.0)\\n\\tfor zzzz@localhost (single-drop); Thu, 22 Aug 2002 12:36:16 +0100 (IST)\\nReceived: from listman.spamassassin.taint.org (listman.spamassassin.taint.org [66.187.233.211]) by\\n    dogma.slashnull.org (8.11.6/8.11.6) with ESMTP id g7MBYrZ04811 for\\n    <zzzz-exmh@spamassassin.taint.org>; Thu, 22 Aug 2002 12:34:53 +0100\\nReceived: from listman.spamassassin.taint.org (localhost.localdomain [1'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_ham = open(os.path.join(HAM_DIR, ham_filenames[0]))\n",
    "sample_ham.read()[:800]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'From 12a1mailbot1@web.de  Thu Aug 22 13:17:22 2002\\nReturn-Path: <12a1mailbot1@web.de>\\nDelivered-To: zzzz@localhost.spamassassin.taint.org\\nReceived: from localhost (localhost [127.0.0.1])\\n\\tby phobos.labs.spamassassin.taint.org (Postfix) with ESMTP id 136B943C32\\n\\tfor <zzzz@localhost>; Thu, 22 Aug 2002 08:17:21 -0400 (EDT)\\nReceived: from mail.webnote.net [193.120.211.219]\\n\\tby localhost with POP3 (fetchmail-5.9.0)\\n\\tfor zzzz@localhost (single-drop); Thu, 22 Aug 2002 13:17:21 +0100 (IST)\\nReceived: from dd_it7 ([210.97.77.167])\\n\\tby webnote.net (8.9.3/8.9.3) with ESMTP id NAA04623\\n\\tfor <zzzz@spamassassin.taint.org>; Thu, 22 Aug 2002 13:09:41 +0100\\nFrom: 12a1mailbot1@web.de\\nReceived: from r-smtp.korea.com - 203.122.2.197 by dd_it7  with Microsoft SMTPSVC(5.5.1775.675.6);\\n\\t Sat, 24 Aug 2002 09:42:10'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_spam = open(os.path.join(SPAM_DIR, spam_filenames[0]))\n",
    "sample_spam.read()[:800]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use `Python email` module to parse these emails and get the content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import email\n",
    "import email.policy\n",
    "\n",
    "def load_email_content(is_spam, filename, path=SPAM_PATH):\n",
    "    directory = \"spam\" if is_spam else \"easy_ham\"\n",
    "    with open(os.path.join(path, directory, filename), \"rb\") as f:\n",
    "        return email.parser.BytesParser(policy=email.policy.default).parse(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Martin A posted:\\nTassos Papadopoulos, the Greek sculptor behind the plan, judged that the\\n limestone of Mount Kerdylio, 70 miles east of Salonika and not far from the\\n Mount Athos monastic community, was ideal for the patriotic sculpture. \\n \\n As well as Alexander's granite features, 240 ft high and 170 ft wide, a\\n museum, a restored amphitheatre and car park for admiring crowds are\\nplanned\\n---------------------\\nSo is this mountain limestone or granite?\\nIf it's limestone, it'll weather pretty fast.\\n\\n------------------------ Yahoo! Groups Sponsor ---------------------~-->\\n4 DVDs Free +s&p Join Now\\nhttp://us.click.yahoo.com/pt6YBB/NXiEAA/mG3HAA/7gSolB/TM\\n---------------------------------------------------------------------~->\\n\\nTo unsubscribe from this group, send an email to:\\nforteana-unsubscribe@egroups.com\\n\\n \\n\\nYour use of Yahoo! Groups is subject to http://docs.yahoo.com/info/terms/\""
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_email_content(is_spam=False, filename=ham_filenames[1]).get_content().strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"1) Fight The Risk of Cancer!\\nhttp://www.adclick.ws/p.cfm?o=315&s=pk007\\n\\n2) Slim Down - Guaranteed to lose 10-12 lbs in 30 days\\nhttp://www.adclick.ws/p.cfm?o=249&s=pk007\\n\\n3) Get the Child Support You Deserve - Free Legal Advice\\nhttp://www.adclick.ws/p.cfm?o=245&s=pk002\\n\\n4) Join the Web's Fastest Growing Singles Community\\nhttp://www.adclick.ws/p.cfm?o=259&s=pk007\\n\\n5) Start Your Private Photo Album Online!\\nhttp://www.adclick.ws/p.cfm?o=283&s=pk007\\n\\nHave a Wonderful Day,\\nOffer Manager\\nPrizeMama\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIf you wish to leave this list please use the link below.\\nhttp://www.qves.com/trim/?ilug@linux.ie%7C17%7C114258\\n\\n\\n-- \\nIrish Linux Users' Group: ilug@linux.ie\\nhttp://www.linux.ie/mailman/listinfo/ilug for (un)subscription information.\\nList maintainer: listmaster@linux.ie\""
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_email_content(is_spam=True, filename=spam_filenames[1]).get_content().strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "ham_emails = [load_email_content(is_spam=False, filename=name) for name in ham_filenames]\n",
    "spam_emails = [load_email_content(is_spam=True, filename=name) for name in spam_filenames]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Many emails are multipart i.e. they have images and attachments which can have their own attachments etc. Let's take a look at types of email structures we have."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_email_structure(email):\n",
    "    if isinstance(email, str):\n",
    "        return email\n",
    "    payload = email.get_payload()\n",
    "    if isinstance(payload, list):\n",
    "        return \"multipart({})\".format(\", \".join([\n",
    "            get_email_structure(sub_email)\n",
    "            for sub_email in payload\n",
    "        ]))\n",
    "    else:\n",
    "        return email.get_content_type()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "def structures_counter(emails):\n",
    "    structures = Counter()\n",
    "    for email in emails:\n",
    "        structure = get_email_structure(email)\n",
    "        structures[structure] += 1\n",
    "    return structures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('text/plain', 2408),\n",
       " ('multipart(text/plain, application/pgp-signature)', 66),\n",
       " ('multipart(text/plain, text/html)', 8),\n",
       " ('multipart(text/plain, text/plain)', 4),\n",
       " ('multipart(text/plain)', 3),\n",
       " ('multipart(text/plain, application/octet-stream)', 2),\n",
       " ('multipart(text/plain, text/enriched)', 1),\n",
       " ('multipart(text/plain, application/ms-tnef, text/plain)', 1),\n",
       " ('multipart(multipart(text/plain, text/plain, text/plain), application/pgp-signature)',\n",
       "  1),\n",
       " ('multipart(text/plain, video/mng)', 1),\n",
       " ('multipart(text/plain, multipart(text/plain))', 1),\n",
       " ('multipart(text/plain, application/x-pkcs7-signature)', 1),\n",
       " ('multipart(text/plain, multipart(text/plain, text/plain), text/rfc822-headers)',\n",
       "  1),\n",
       " ('multipart(text/plain, multipart(text/plain, text/plain), multipart(multipart(text/plain, application/x-pkcs7-signature)))',\n",
       "  1),\n",
       " ('multipart(text/plain, application/x-java-applet)', 1)]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structures_counter(ham_emails).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('text/plain', 218),\n",
       " ('text/html', 183),\n",
       " ('multipart(text/plain, text/html)', 45),\n",
       " ('multipart(text/html)', 20),\n",
       " ('multipart(text/plain)', 19),\n",
       " ('multipart(multipart(text/html))', 5),\n",
       " ('multipart(text/plain, image/jpeg)', 3),\n",
       " ('multipart(text/html, application/octet-stream)', 2),\n",
       " ('multipart(text/plain, application/octet-stream)', 1),\n",
       " ('multipart(text/html, text/plain)', 1),\n",
       " ('multipart(multipart(text/html), application/octet-stream, image/jpeg)', 1),\n",
       " ('multipart(multipart(text/plain, text/html), image/gif)', 1),\n",
       " ('multipart/alternative', 1)]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structures_counter(spam_emails).most_common()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It looks like Spam emails have lot of HTML while ham emails are plain text. We also have 66 ham emails signed by PGP, but no such spam email.\n",
    "\n",
    "Let's explore email headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Return-Path  :  <12a1mailbot1@web.de>\n",
      "Delivered-To  :  zzzz@localhost.spamassassin.taint.org\n",
      "Received  :  from localhost (localhost [127.0.0.1])\tby phobos.labs.spamassassin.taint.org (Postfix) with ESMTP id 136B943C32\tfor <zzzz@localhost>; Thu, 22 Aug 2002 08:17:21 -0400 (EDT)\n",
      "Received  :  from mail.webnote.net [193.120.211.219]\tby localhost with POP3 (fetchmail-5.9.0)\tfor zzzz@localhost (single-drop); Thu, 22 Aug 2002 13:17:21 +0100 (IST)\n",
      "Received  :  from dd_it7 ([210.97.77.167])\tby webnote.net (8.9.3/8.9.3) with ESMTP id NAA04623\tfor <zzzz@spamassassin.taint.org>; Thu, 22 Aug 2002 13:09:41 +0100\n",
      "From  :  12a1mailbot1@web.de\n",
      "Received  :  from r-smtp.korea.com - 203.122.2.197 by dd_it7  with Microsoft SMTPSVC(5.5.1775.675.6);\t Sat, 24 Aug 2002 09:42:10 +0900\n",
      "To  :  dcek1a1@netsgo.com\n",
      "Subject  :  Life Insurance - Why Pay More?\n",
      "Date  :  Wed, 21 Aug 2002 20:31:57 -1600\n",
      "MIME-Version  :  1.0\n",
      "Message-ID  :  <0103c1042001882DD_IT7@dd_it7>\n",
      "Content-Type  :  text/html; charset=\"iso-8859-1\"\n",
      "Content-Transfer-Encoding  :  quoted-printable\n"
     ]
    }
   ],
   "source": [
    "for header, value in spam_emails[0].items():\n",
    "    print(header, \" : \", value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Although, lot of information here, lets focus on subject of the email for now."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Life Insurance - Why Pay More?'"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spam_emails[0]['Subject']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Time to split into train test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = np.array(ham_emails + spam_emails)\n",
    "y = np.array([0] * len(ham_emails) + [1] * len(spam_emails))\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we will start preprocessing the data.First let's convert HTML to plain text. We will use [BeautifulSoup](https://pypi.org/project/beautifulsoup4/) for that. The below function does that, it also removes the extra new lines in the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "\n",
    "def html_to_plain_text(html):\n",
    "    soup = BeautifulSoup(html, \"html.parser\")\n",
    "    text = re.sub(r'(\\s*\\n)+', '\\n', soup.text, flags=re.M | re.S)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<html>\n",
      "\n",
      "<head>\n",
      "<title>Home Page</title>\n",
      "</head>\n",
      "\n",
      "<body>\n",
      "\n",
      "<p align=\"center\"><font color=\"#000000\" face=\"Arial\" size=\"+0\"><b><IMG SRC=\"http://mail4.mortgages101.net/logo.php?id=88&id2=1143953\"></p>\n",
      "\n",
      "<p align=\"center\">If this promotion has reached you in error and you would prefer not to\n",
      "receive marketing messages from us, please send an email to&nbsp; <a\n",
      "href=\"mailto:cease-and-desist@mortgages101.net\">cease-and-desist@mortgages101.net</a>\n",
      "&nbsp; (all one word, no spaces) giving us the email address in question or call\n",
      "1-888-748-7751 for further assistance.</p>\n",
      "\n",
      "<p align=\"center\"><u>Gain access to a</b></font><font size=\"+1\" color=\"#000000\"\n",
      "face=\"Arial\"> <i><b>Vast Network Of Qualified Lenders at Nationwide Network!</b></i></font></u></p>\n",
      "\n",
      "<p align=\"center\"><font color=\"#000000\" face=\"Arial\">This is a zero-cost service which\n",
      "enables you to shop for a mortgage conveniently from your home computer. &nbsp; Our\n",
      "nationwide database will give you access to lenders with a variety of loan program\n"
     ]
    }
   ],
   "source": [
    "html_spam_emails = [email for email in X_train[y_train==1]\n",
    "                   if get_email_structure(email) == 'text/html']\n",
    "\n",
    "sample_html_spam = html_spam_emails[9]\n",
    "print(sample_html_spam.get_content().strip()[:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Home Page\n",
      "If this promotion has reached you in error and you would prefer not to\n",
      "receive marketing messages from us, please send an email to  cease-and-desist@mortgages101.net\n",
      "  (all one word, no spaces) giving us the email address in question or call\n",
      "1-888-748-7751 for further assistance.\n",
      "Gain access to a Vast Network Of Qualified Lenders at Nationwide Network!\n",
      "This is a zero-cost service which\n",
      "enables you to shop for a mortgage conveniently from your home computer.   Our\n",
      "nationwide database will give you access to lenders with a variety of loan program\n"
     ]
    }
   ],
   "source": [
    "print(html_to_plain_text(sample_html_spam.get_content().strip()[:1000]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's combine everything together, so we can convert an email to text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "def email_to_text(email):\n",
    "    html = None\n",
    "    for part in email.walk():\n",
    "        ctype = part.get_content_type()\n",
    "        if not ctype in (\"text/plain\", \"text/html\"):\n",
    "            continue\n",
    "        try:\n",
    "            content = part.get_content()\n",
    "        except:\n",
    "            content = str(part.get_payload())\n",
    "        if ctype == \"text/plain\":\n",
    "            return content\n",
    "        else:\n",
    "            html = content\n",
    "    if html:\n",
    "        return html_to_plain_text(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Home Page\n",
      "If this promotion has reached you in error and you would prefer not to\n",
      "receive marketing  ...\n"
     ]
    }
   ],
   "source": [
    "print(email_to_text(sample_html_spam)[:100], \"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computations => comput\n",
      "Computation => comput\n",
      "Computing => comput\n",
      "Computed => comput\n",
      "Compute => comput\n",
      "Compulsive => compuls\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    import nltk\n",
    "\n",
    "    stemmer = nltk.PorterStemmer()\n",
    "    for word in (\"Computations\", \"Computation\", \"Computing\", \"Computed\", \"Compute\", \"Compulsive\"):\n",
    "        print(word, \"=>\", stemmer.stem(word))\n",
    "except ImportError:\n",
    "    print(\"Error: stemming requires the NLTK module.\")\n",
    "    stemmer = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['example.com', 'github.com', 'https://youtube.com']\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    import urlextract \n",
    "    url_extractor = urlextract.URLExtract()\n",
    "    print(url_extractor.find_urls(\"This is an example.com domain which hosts the site on github.com and also on https://youtube.com\"))\n",
    "except ImportError:\n",
    "    print(\"Error replacing URLs requires the urlextract module\")\n",
    "    url_extractor = None\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we need to count how often a word appears in the email, Spam emails will likely have some words (e.g. loan, or porn) than Ham emails. let's build a transformer to convert email content to a dict of words with their count."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class EmailToWordCounterTransformer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, strip_headers=True, lower_case=True, remove_punctuation=True,\n",
    "               replace_urls=True, replace_numbers=True, stemming=True):\n",
    "        self.strip_headers = strip_headers\n",
    "        self.lower_case = lower_case\n",
    "        self.remove_punctuation = remove_punctuation\n",
    "        self.replace_urls = replace_urls\n",
    "        self.replace_numbers = replace_numbers\n",
    "        self.stemming = stemming\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X, y=None):\n",
    "        X_transformed = []\n",
    "        for email in X:\n",
    "            text = email_to_text(email) or \"\"\n",
    "            if self.lower_case:\n",
    "                text = text.lower()\n",
    "            if self.replace_urls and url_extractor is not None:\n",
    "                urls = list(set(url_extractor.find_urls(text)))\n",
    "                urls.sort(key=lambda url: len(url), reverse=True)\n",
    "                for url in urls:\n",
    "                    text = text.replace(url, \" URL \")\n",
    "            if self.replace_numbers:\n",
    "                text = re.sub(r'\\d+(?:\\.\\d*(?:[eE]\\d+))?', ' NUMBER ', text)\n",
    "            if self.remove_punctuation:\n",
    "                text = re.sub(r'\\W+', ' ', text, flags=re.M)\n",
    "            word_counts = Counter(text.split())\n",
    "            if self.stemming and stemmer is not None:\n",
    "                stemmed_word_counts = Counter()\n",
    "                for word, count in word_counts.items():\n",
    "                    stemmed_word = stemmer.stem(word)\n",
    "                    stemmed_word_counts[stemmed_word] += count\n",
    "                word_counts = stemmed_word_counts\n",
    "            X_transformed.append(word_counts)\n",
    "        return np.array(X_transformed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run this on few emails."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([Counter({'chuck': 1, 'murcko': 1, 'wrote': 1, 'stuff': 1, 'yawn': 1, 'r': 1}),\n",
       "       Counter({'the': 11, 'of': 9, 'and': 8, 'all': 3, 'christian': 3, 'to': 3, 'by': 3, 'jefferson': 2, 'i': 2, 'have': 2, 'superstit': 2, 'one': 2, 'on': 2, 'been': 2, 'ha': 2, 'half': 2, 'rogueri': 2, 'teach': 2, 'jesu': 2, 'some': 1, 'interest': 1, 'quot': 1, 'url': 1, 'thoma': 1, 'examin': 1, 'known': 1, 'word': 1, 'do': 1, 'not': 1, 'find': 1, 'in': 1, 'our': 1, 'particular': 1, 'redeem': 1, 'featur': 1, 'they': 1, 'are': 1, 'alik': 1, 'found': 1, 'fabl': 1, 'mytholog': 1, 'million': 1, 'innoc': 1, 'men': 1, 'women': 1, 'children': 1, 'sinc': 1, 'introduct': 1, 'burnt': 1, 'tortur': 1, 'fine': 1, 'imprison': 1, 'what': 1, 'effect': 1, 'thi': 1, 'coercion': 1, 'make': 1, 'world': 1, 'fool': 1, 'other': 1, 'hypocrit': 1, 'support': 1, 'error': 1, 'over': 1, 'earth': 1, 'six': 1, 'histor': 1, 'american': 1, 'john': 1, 'e': 1, 'remsburg': 1, 'letter': 1, 'william': 1, 'short': 1, 'again': 1, 'becom': 1, 'most': 1, 'pervert': 1, 'system': 1, 'that': 1, 'ever': 1, 'shone': 1, 'man': 1, 'absurd': 1, 'untruth': 1, 'were': 1, 'perpetr': 1, 'upon': 1, 'a': 1, 'larg': 1, 'band': 1, 'dupe': 1, 'import': 1, 'led': 1, 'paul': 1, 'first': 1, 'great': 1, 'corrupt': 1}),\n",
       "       Counter({'url': 4, 's': 3, 'group': 3, 'to': 3, 'in': 2, 'forteana': 2, 'martin': 2, 'an': 2, 'and': 2, 'we': 2, 'is': 2, 'yahoo': 2, 'unsubscrib': 2, 'y': 1, 'adamson': 1, 'wrote': 1, 'for': 1, 'altern': 1, 'rather': 1, 'more': 1, 'factual': 1, 'base': 1, 'rundown': 1, 'on': 1, 'hamza': 1, 'career': 1, 'includ': 1, 'hi': 1, 'belief': 1, 'that': 1, 'all': 1, 'non': 1, 'muslim': 1, 'yemen': 1, 'should': 1, 'be': 1, 'murder': 1, 'outright': 1, 'know': 1, 'how': 1, 'unbias': 1, 'memri': 1, 'don': 1, 't': 1, 'html': 1, 'rob': 1, 'sponsor': 1, 'number': 1, 'dvd': 1, 'free': 1, 'p': 1, 'join': 1, 'now': 1, 'from': 1, 'thi': 1, 'send': 1, 'email': 1, 'egroup': 1, 'com': 1, 'your': 1, 'use': 1, 'of': 1, 'subject': 1})],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_few = X_train[:3]\n",
    "X_few_wordcounts = EmailToWordCounterTransformer().fit_transform(X_few)\n",
    "X_few_wordcounts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Awesome! This looks right. Now, let's convert them to vectors. FOr this, we will build another transformer whose `fit()` method will build the vocabulary (a ordered list of most common words) and whose `transform()` method will use the vocabulary to convert word counts to vectors. The output will be a sparse matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix\n",
    "\n",
    "class WordCounterToVectorTransformer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, vocabulary_size=1000):\n",
    "        self.vocabulary_size = vocabulary_size\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        total_count = Counter()\n",
    "        for word_count in X:\n",
    "            for word, count in word_count.items():\n",
    "                total_count[word] += min(count, 10)\n",
    "        most_common = total_count.most_common()[:self.vocabulary_size]\n",
    "        self.most_common_ = most_common\n",
    "        self.vocabulary_ = {word: index + 1 for index, (word, count) in enumerate(most_common)}\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X, y=None):\n",
    "        rows = []\n",
    "        cols = []\n",
    "        data = []\n",
    "        for row, word_count in enumerate(X):\n",
    "            for word, count in word_count.items():\n",
    "                rows.append(row)\n",
    "                cols.append(self.vocabulary_.get(word, 0))\n",
    "                data.append(count)\n",
    "        return csr_matrix((data, (rows, cols)), shape=(len(X), self.vocabulary_size + 1))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<3x11 sparse matrix of type '<class 'numpy.intc'>'\n",
       "\twith 20 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_transformer = WordCounterToVectorTransformer(vocabulary_size=10)\n",
    "X_few_vectors = vocab_transformer.fit_transform(X_few_wordcounts)\n",
    "X_few_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 6,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [99, 11,  9,  8,  3,  1,  3,  1,  3,  2,  3],\n",
       "       [67,  0,  1,  2,  3,  4,  1,  2,  0,  1,  0]], dtype=int32)"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_few_vectors.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'the': 1,\n",
       " 'of': 2,\n",
       " 'and': 3,\n",
       " 'to': 4,\n",
       " 'url': 5,\n",
       " 'all': 6,\n",
       " 'in': 7,\n",
       " 'christian': 8,\n",
       " 'on': 9,\n",
       " 'by': 10}"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_transformer.vocabulary_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What does this matrix mean? Well, let's analyze the second row.\n",
    "\n",
    "- First number in column (value=99) represents the number of words which are not part of vocabulary (Check code, what goes index 0?\n",
    "- Second number in column (value=11) represents the first word in the vocabulary is present 11 times in this email, that is the word *the*.\n",
    "- Third number in column (value=9) represents that second word in the vocabulary is present 9 times in this email, and that word is *of*\n",
    "- and so on..\n",
    "\n",
    "Time to build a pipeline."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "preprocess_pipeline = Pipeline([\n",
    "    (\"email_to_wordcount\", EmailToWordCounterTransformer()),\n",
    "    (\"wordcount_to_vector\", WordCounterToVectorTransformer())\n",
    "])\n",
    "\n",
    "X_train_transformed = preprocess_pipeline.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally the classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done   3 out of   3 | elapsed:    9.5s finished\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "log_clf = LogisticRegression(solver=\"lbfgs\", random_state=42)\n",
    "score = cross_val_score(log_clf, X_train_transformed, y_train, cv=3, verbose=3, n_jobs=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9829166666666667"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "98% is not bad for the first try, ofcouse the dataset was \"easy\". The variant of datasets presents which makes this problem harder. For those, we will have to try multiple models to reach this accuracy, select the best ones, and fine-tune them. Let's look at other important metrics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision : 93.93939393939394\n",
      "Recall    : 97.89473684210527\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\envs\\ml\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:938: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score\n",
    "\n",
    "X_test_transformed = preprocess_pipeline.transform(X_test)\n",
    "log_clf = LogisticRegression(solver=\"lbfgs\", random_state=42)\n",
    "log_clf.fit(X_train_transformed, y_train)\n",
    "\n",
    "y_pred = log_clf.predict(X_test_transformed)\n",
    "\n",
    "print(F\"Precision : {100 * precision_score(y_test, y_pred)}\")\n",
    "print(F\"Recall    : {100 * recall_score(y_test, y_pred)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}