SocialUnrest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import html.parser as htmlParser\n",
    "import re\n",
    "from datetime import datetime\n",
    "from langdetect import detect\n",
    "import nltk\n",
    "from nltk.corpus import sentiwordnet as swn\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, f1_score\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import BernoulliNB\n",
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Open data file for Charlottesville event and add some useful columns to the dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# open file\n",
    "cVille_df = pd.read_json('./Datasets/charlottesville.json')\n",
    "\n",
    "# add ids\n",
    "cVille_df['id'] = [url.rsplit('/', 1)[-1] for url in cVille_df['permalink']]\n",
    "\n",
    "# get number of mentions instead of actual values\n",
    "cVille_df['mention_count'] = [(len(mentions.split(' ')) if mentions != \"\" else 0) for mentions in cVille_df['mentions']]\n",
    "\n",
    "# add hashtag count column as well\n",
    "cVille_df['hashtag_count'] = [(len(hashtags.split(' ')) if hashtags != \"\" else 0)  for hashtags in cVille_df['hashtags'] ]\n",
    "\n",
    "# drop username and geo\n",
    "cVille_df = cVille_df.drop(['geo', 'username', 'permalink', 'mentions'], axis=1)\n",
    "\n",
    "# add lang column\n",
    "cVille_df['lang'] = [detect(text) for text in cVille_df['text']]\n",
    "\n",
    "# drop non-english rows\n",
    "cVille_df = cVille_df[cVille_df.lang == 'en']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Open data file for WomenMarch event and add some useful columns to the dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# open file\n",
    "womenMarch_df = pd.read_json('./Datasets/womenmarch.json')\n",
    "\n",
    "# add ids\n",
    "womenMarch_df['id'] = [url.rsplit('/', 1)[-1] for url in womenMarch_df['permalink']]\n",
    "\n",
    "# get number of mentions instead of actual values\n",
    "womenMarch_df['mention_count'] = [(len(mentions.split(' ')) if mentions != \"\" else 0) for mentions in womenMarch_df['mentions']]\n",
    "\n",
    "# add hashtag count column as well\n",
    "womenMarch_df['hashtag_count'] = [(len(hashtags.split(' ')) if hashtags != \"\" else 0)  for hashtags in womenMarch_df['hashtags'] ]\n",
    "\n",
    "# drop username and geo\n",
    "womenMarch_df = womenMarch_df.drop(['geo', 'username', 'permalink', 'mentions'], axis=1)\n",
    "\n",
    "# add lang column\n",
    "womenMarch_df['lang'] = [detect(text) for text in womenMarch_df['text']]\n",
    "\n",
    "# drop non-english rows\n",
    "womenMarch_df = womenMarch_df[womenMarch_df.lang == 'en']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some more preprocessing of data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def containsChar(text, pattern):\n",
    "    for word in text.split(' '):\n",
    "        if word == pattern: \n",
    "            return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preprocess data function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This method preprocesses the data in given dataframe\n",
    "# wasViolent = True indicates violent event and False indicates non-violent event \n",
    "\n",
    "def preprocessData(df, wasViolent):\n",
    "    \n",
    "    # convert html elements to readable format (also convert text to lowercase)\n",
    "    filteredText = [htmlParser.unescape(text.lower()) for text in df['text']]\n",
    "\n",
    "    # remove urls (because urls conatain spaces that make it difficult, this will stop at ellipsis or # or @ or end of line)\n",
    "    # not 100% accurate but decent approach\n",
    "    url_pattern = re.compile(\"http(.+?)(@\\S+|#\\S+|\\\\u2026|$)\")\n",
    "    filteredText = [re.sub(url_pattern, \"\", text) for text in filteredText]\n",
    "\n",
    "    # remove usernames and mentions\n",
    "    username_pattern = re.compile(\"@\\S+\")\n",
    "    filteredText = [re.sub(username_pattern, \"\", text) for text in filteredText]\n",
    "\n",
    "    # remove hashtags\n",
    "    hashtag_pattern = re.compile(\"#\\S+\")\n",
    "    filteredText = [re.sub(hashtag_pattern, \"\", text) for text in filteredText]\n",
    "\n",
    "    # emoticon handling (replace positive_emojis with pos_emoji and negative_emojis with neg_emoji keyword)\n",
    "    pos_emojis = [':-)', ':)', '(:', '(-:', ':-D', ':D', 'X-D', 'XD', 'xD', '<3', ':*', ';-)', ';)', ';-D', ';D', '(;', '(-;']\n",
    "    filteredText = [' '.join([('pos_emoji' if word in pos_emojis else word) for word in text.split(' ')]) for text in filteredText]\n",
    "\n",
    "    # add two new features (pos_emoji and neg_emoji - 1 for presence and 0 for absence)\n",
    "    df['pos_emoji'] = [(1 if containsChar(text, 'pos_emoji') else 0) for text in filteredText]\n",
    "    df['neg_emoji'] = [(1 if containsChar(text, 'neg_emoji') else 0) for text in filteredText]\n",
    "    \n",
    "    neg_emojis = [\":-(\", \":(\", \"):\", \")-:\", \":,(\", \":'(\", ':\"(', \":((\"]\n",
    "    filteredText = [' '.join([('neg_emoji' if word in neg_emojis else word) for word in text.split(' ')]) for text in filteredText]\n",
    "\n",
    "    # handle punctuations (add punc_excl=1 if ! exists else 0, same with punc_ques and ?)\n",
    "    df['punc_excl'] = [(1 if containsChar(text, '!') else 0) for text in filteredText]\n",
    "    df['punc_ques'] = [(1 if containsChar(text, '?') else 0) for text in filteredText]\n",
    "    \n",
    "    # remove pictures\n",
    "    pic_pattern = re.compile(\"pic.twitter.com(.+)(/s|$)\")\n",
    "    filteredText = [re.sub(pic_pattern, \"\", text) for text in filteredText]\n",
    "\n",
    "    # remove question marks and dollar signs\n",
    "    removableChars = ['?', '$']\n",
    "    filteredText = [' '.join([('' if word in removableChars else word) for word in text.split(' ')]) for text in filteredText]\n",
    "\n",
    "    # remove random words (if still there is : or emoji then ignore them)\n",
    "    random_pattern = re.compile(\"(&|:|!|,|;|)\", re.I)\n",
    "    filteredText = [re.sub(random_pattern, \"\", text) for text in filteredText]\n",
    "\n",
    "    # handle repeating characters (replace characters repeating more than twice as two characters)\n",
    "    repeated_pattern = re.compile(r\"(.)\\1{1,}\")\n",
    "    filteredText = [re.sub(repeated_pattern, r'\\1\\1', text) for text in filteredText]\n",
    "\n",
    "    # add label\n",
    "    df['violent'] = [1 if wasViolent else 0] * df.shape[0]\n",
    "\n",
    "    # add filteredText column to dataframe\n",
    "    df['filteredText'] = filteredText\n",
    "    \n",
    "    return df[['id', 'date', 'filteredText', 'hashtags', 'favorites', 'retweets', 'mention_count', 'hashtag_count', 'pos_emoji', 'neg_emoji', 'punc_excl', 'punc_ques', 'violent']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check number of valid samples in English"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(154020, 9)\n",
      "(109967, 9)\n"
     ]
    }
   ],
   "source": [
    "print(cVille_df.shape)\n",
    "print(womenMarch_df.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preprocess both Charlottesville and WomenMarch data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cVille_preprocessed_df = preprocessData(cVille_df, wasViolent=True)\n",
    "womenMarch_preprocessed_df = preprocessData(womenMarch_df, wasViolent=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>filteredText</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>favorites</th>\n",
       "      <th>retweets</th>\n",
       "      <th>mention_count</th>\n",
       "      <th>hashtag_count</th>\n",
       "      <th>pos_emoji</th>\n",
       "      <th>neg_emoji</th>\n",
       "      <th>punc_excl</th>\n",
       "      <th>punc_ques</th>\n",
       "      <th>violent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>884925177917845504</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>de blasio’s trip was approved for ‘city purpos...</td>\n",
       "      <td>#commies #antifa</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>884924281951576064</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>alt-right trump-supporting neo-nazi confirmed.</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>884923880938328066</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>this. kills me. i remember seeing  around duri...</td>\n",
       "      <td>#RobertGrodt #Occupy #BijiRojava #antifa</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>884923588960280576</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>is resorting to violence and unrest to silenc...</td>\n",
       "      <td>#antifa</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>884922175748059136</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>neo-nazi  july 12th internet-wide day of act...</td>\n",
       "      <td>#Politics #Collusion #NetNeutrality</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id       date  \\\n",
       "0  884925177917845504 2017-07-11   \n",
       "1  884924281951576064 2017-07-11   \n",
       "2  884923880938328066 2017-07-11   \n",
       "3  884923588960280576 2017-07-11   \n",
       "4  884922175748059136 2017-07-11   \n",
       "\n",
       "                                        filteredText  \\\n",
       "0  de blasio’s trip was approved for ‘city purpos...   \n",
       "1     alt-right trump-supporting neo-nazi confirmed.   \n",
       "2  this. kills me. i remember seeing  around duri...   \n",
       "3   is resorting to violence and unrest to silenc...   \n",
       "4    neo-nazi  july 12th internet-wide day of act...   \n",
       "\n",
       "                                   hashtags  favorites  retweets  \\\n",
       "0                          #commies #antifa          0         2   \n",
       "1                                                    0         0   \n",
       "2  #RobertGrodt #Occupy #BijiRojava #antifa          7         4   \n",
       "3                                   #antifa          0         0   \n",
       "4       #Politics #Collusion #NetNeutrality          0         1   \n",
       "\n",
       "   mention_count  hashtag_count  pos_emoji  neg_emoji  punc_excl  punc_ques  \\\n",
       "0              0              2          0          0          0          0   \n",
       "1              0              0          0          0          0          0   \n",
       "2              0              4          0          0          0          0   \n",
       "3              0              1          0          0          0          0   \n",
       "4              1              3          0          0          0          0   \n",
       "\n",
       "   violent  \n",
       "0        1  \n",
       "1        1  \n",
       "2        1  \n",
       "3        1  \n",
       "4        1  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cVille_preprocessed_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>filteredText</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>favorites</th>\n",
       "      <th>retweets</th>\n",
       "      <th>mention_count</th>\n",
       "      <th>hashtag_count</th>\n",
       "      <th>pos_emoji</th>\n",
       "      <th>neg_emoji</th>\n",
       "      <th>punc_excl</th>\n",
       "      <th>punc_ques</th>\n",
       "      <th>violent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>921443913398972416</td>\n",
       "      <td>2017-10-20</td>\n",
       "      <td>can we make our message any clearer?</td>\n",
       "      <td>#MeToo #EndRapeCulture #NotOkay #WhyIMarch</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>921204515738546176</td>\n",
       "      <td>2017-10-19</td>\n",
       "      <td>anyone who doesn't support affordable  is not ...</td>\n",
       "      <td>#birthcontrol #prolife #ProofIsInThePudding #W...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>921100053720256514</td>\n",
       "      <td>2017-10-19</td>\n",
       "      <td>each and every day for</td>\n",
       "      <td>#WhyIMarch #RacialJustice #M4RJ #BlackLivesMatter</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>920843500781953024</td>\n",
       "      <td>2017-10-18</td>\n",
       "      <td>because  deserve a livable future.</td>\n",
       "      <td>#whyimarch #youth</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>920765197605855232</td>\n",
       "      <td>2017-10-18</td>\n",
       "      <td>started a conversation with my daughter  . i ...</td>\n",
       "      <td>#MeToo #WhyIMarch #ShePersisted</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id       date  \\\n",
       "0  921443913398972416 2017-10-20   \n",
       "1  921204515738546176 2017-10-19   \n",
       "2  921100053720256514 2017-10-19   \n",
       "3  920843500781953024 2017-10-18   \n",
       "4  920765197605855232 2017-10-18   \n",
       "\n",
       "                                        filteredText  \\\n",
       "0             can we make our message any clearer?     \n",
       "1  anyone who doesn't support affordable  is not ...   \n",
       "2                           each and every day for     \n",
       "3                because  deserve a livable future.    \n",
       "4   started a conversation with my daughter  . i ...   \n",
       "\n",
       "                                            hashtags  favorites  retweets  \\\n",
       "0         #MeToo #EndRapeCulture #NotOkay #WhyIMarch          0         1   \n",
       "1  #birthcontrol #prolife #ProofIsInThePudding #W...          1         1   \n",
       "2  #WhyIMarch #RacialJustice #M4RJ #BlackLivesMatter          0         2   \n",
       "3                                  #whyimarch #youth          0         1   \n",
       "4                    #MeToo #WhyIMarch #ShePersisted          2         3   \n",
       "\n",
       "   mention_count  hashtag_count  pos_emoji  neg_emoji  punc_excl  punc_ques  \\\n",
       "0              0              4          0          0          0          0   \n",
       "1              0              5          0          0          0          0   \n",
       "2              0              4          0          0          0          0   \n",
       "3              1              2          0          0          0          0   \n",
       "4              1              3          0          0          0          0   \n",
       "\n",
       "   violent  \n",
       "0        0  \n",
       "1        0  \n",
       "2        0  \n",
       "3        0  \n",
       "4        0  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "womenMarch_preprocessed_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# write preprocessed data back to to the disk\n",
    "cVille_preprocessed_df.to_csv('./Datasets/charlottesville_preprocessed.csv', sep='\\t', index=False)\n",
    "womenMarch_preprocessed_df.to_csv('./Datasets/womenMarch_preprocessed.csv', sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read Charlottesville and Women March preprocessed data from the disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cVille_preprocessed_df = pd.read_csv('./Datasets/charlottesville_preprocessed.csv', sep='\\t')\n",
    "womenMarch_preprocessed_df = pd.read_csv('./Datasets/womenMarch_preprocessed.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Handle NaN values. Replace them with empty string in filteredText column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 154020 entries, 0 to 154019\n",
      "Data columns (total 13 columns):\n",
      "id               154020 non-null int64\n",
      "date             154020 non-null object\n",
      "filteredText     153959 non-null object\n",
      "hashtags         143978 non-null object\n",
      "favorites        154020 non-null int64\n",
      "retweets         154020 non-null int64\n",
      "mention_count    154020 non-null int64\n",
      "hashtag_count    154020 non-null int64\n",
      "pos_emoji        154020 non-null int64\n",
      "neg_emoji        154020 non-null int64\n",
      "punc_excl        154020 non-null int64\n",
      "punc_ques        154020 non-null int64\n",
      "violent          154020 non-null int64\n",
      "dtypes: int64(10), object(3)\n",
      "memory usage: 15.3+ MB\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 154020 entries, 0 to 154019\n",
      "Data columns (total 13 columns):\n",
      "id               154020 non-null int64\n",
      "date             154020 non-null object\n",
      "filteredText     154020 non-null object\n",
      "hashtags         154020 non-null object\n",
      "favorites        154020 non-null int64\n",
      "retweets         154020 non-null int64\n",
      "mention_count    154020 non-null int64\n",
      "hashtag_count    154020 non-null int64\n",
      "pos_emoji        154020 non-null int64\n",
      "neg_emoji        154020 non-null int64\n",
      "punc_excl        154020 non-null int64\n",
      "punc_ques        154020 non-null int64\n",
      "violent          154020 non-null int64\n",
      "dtypes: int64(10), object(3)\n",
      "memory usage: 15.3+ MB\n"
     ]
    }
   ],
   "source": [
    "cVille_preprocessed_df.info()\n",
    "cVille_preprocessed_df['filteredText'].fillna(\"\", inplace=True)\n",
    "cVille_preprocessed_df['hashtags'].fillna(\"\", inplace=True)\n",
    "cVille_preprocessed_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 109967 entries, 0 to 109966\n",
      "Data columns (total 13 columns):\n",
      "id               109967 non-null int64\n",
      "date             109967 non-null object\n",
      "filteredText     109828 non-null object\n",
      "hashtags         109967 non-null object\n",
      "favorites        109967 non-null int64\n",
      "retweets         109967 non-null int64\n",
      "mention_count    109967 non-null int64\n",
      "hashtag_count    109967 non-null int64\n",
      "pos_emoji        109967 non-null int64\n",
      "neg_emoji        109967 non-null int64\n",
      "punc_excl        109967 non-null int64\n",
      "punc_ques        109967 non-null int64\n",
      "violent          109967 non-null int64\n",
      "dtypes: int64(10), object(3)\n",
      "memory usage: 10.9+ MB\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 109967 entries, 0 to 109966\n",
      "Data columns (total 13 columns):\n",
      "id               109967 non-null int64\n",
      "date             109967 non-null object\n",
      "filteredText     109967 non-null object\n",
      "hashtags         109967 non-null object\n",
      "favorites        109967 non-null int64\n",
      "retweets         109967 non-null int64\n",
      "mention_count    109967 non-null int64\n",
      "hashtag_count    109967 non-null int64\n",
      "pos_emoji        109967 non-null int64\n",
      "neg_emoji        109967 non-null int64\n",
      "punc_excl        109967 non-null int64\n",
      "punc_ques        109967 non-null int64\n",
      "violent          109967 non-null int64\n",
      "dtypes: int64(10), object(3)\n",
      "memory usage: 10.9+ MB\n"
     ]
    }
   ],
   "source": [
    "womenMarch_preprocessed_df.info()\n",
    "womenMarch_preprocessed_df['hashtags'].fillna(\"\", inplace=True)\n",
    "womenMarch_preprocessed_df['filteredText'].fillna(\"\", inplace=True)\n",
    "womenMarch_preprocessed_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>filteredText</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>favorites</th>\n",
       "      <th>retweets</th>\n",
       "      <th>mention_count</th>\n",
       "      <th>hashtag_count</th>\n",
       "      <th>pos_emoji</th>\n",
       "      <th>neg_emoji</th>\n",
       "      <th>punc_excl</th>\n",
       "      <th>punc_ques</th>\n",
       "      <th>violent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>921443913398972416</td>\n",
       "      <td>2017-10-20</td>\n",
       "      <td>can we make our message any clearer?</td>\n",
       "      <td>#MeToo #EndRapeCulture #NotOkay #WhyIMarch</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>921204515738546176</td>\n",
       "      <td>2017-10-19</td>\n",
       "      <td>anyone who doesn't support affordable  is not ...</td>\n",
       "      <td>#birthcontrol #prolife #ProofIsInThePudding #W...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>921100053720256514</td>\n",
       "      <td>2017-10-19</td>\n",
       "      <td>each and every day for</td>\n",
       "      <td>#WhyIMarch #RacialJustice #M4RJ #BlackLivesMatter</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>920843500781953024</td>\n",
       "      <td>2017-10-18</td>\n",
       "      <td>because  deserve a livable future.</td>\n",
       "      <td>#whyimarch #youth</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>920765197605855232</td>\n",
       "      <td>2017-10-18</td>\n",
       "      <td>started a conversation with my daughter  . i ...</td>\n",
       "      <td>#MeToo #WhyIMarch #ShePersisted</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id        date  \\\n",
       "0  921443913398972416  2017-10-20   \n",
       "1  921204515738546176  2017-10-19   \n",
       "2  921100053720256514  2017-10-19   \n",
       "3  920843500781953024  2017-10-18   \n",
       "4  920765197605855232  2017-10-18   \n",
       "\n",
       "                                        filteredText  \\\n",
       "0             can we make our message any clearer?     \n",
       "1  anyone who doesn't support affordable  is not ...   \n",
       "2                           each and every day for     \n",
       "3                because  deserve a livable future.    \n",
       "4   started a conversation with my daughter  . i ...   \n",
       "\n",
       "                                            hashtags  favorites  retweets  \\\n",
       "0         #MeToo #EndRapeCulture #NotOkay #WhyIMarch          0         1   \n",
       "1  #birthcontrol #prolife #ProofIsInThePudding #W...          1         1   \n",
       "2  #WhyIMarch #RacialJustice #M4RJ #BlackLivesMatter          0         2   \n",
       "3                                  #whyimarch #youth          0         1   \n",
       "4                    #MeToo #WhyIMarch #ShePersisted          2         3   \n",
       "\n",
       "   mention_count  hashtag_count  pos_emoji  neg_emoji  punc_excl  punc_ques  \\\n",
       "0              0              4          0          0          0          0   \n",
       "1              0              5          0          0          0          0   \n",
       "2              0              4          0          0          0          0   \n",
       "3              1              2          0          0          0          0   \n",
       "4              1              3          0          0          0          0   \n",
       "\n",
       "   violent  \n",
       "0        0  \n",
       "1        0  \n",
       "2        0  \n",
       "3        0  \n",
       "4        0  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "womenMarch_preprocessed_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# below is part of preprocessing\n",
    "def feature_reduced_text(text, stop_words, lemmatizer):\n",
    "    # tokenization\n",
    "    tokens = word_tokenize(text)\n",
    "\n",
    "    # remove stopwords from text\n",
    "    tokens = [token for token in tokens if token not in stop_words]\n",
    "\n",
    "    # lemmatization\n",
    "    tokens = [lemmatizer.lemmatize(token) for token in tokens]\n",
    "\n",
    "    # pos tagging\n",
    "    pos_tags = nltk.pos_tag(tokens)\n",
    "    \n",
    "    # return only relevant words (adjective, verb, adverb, noun with all of their variations)\n",
    "    return ' '.join([tag[0] for tag in pos_tags if tag[1] in ['JJ', 'JJR', 'JJS', 'NN', 'NNP', 'NNS', 'NNP', 'NNPS', 'RB', 'RBR', 'RBS', 'VB', 'VBG', 'VBD', 'VBN', 'VBP', 'VBZ']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "stops = set(stopwords.words('english'))\n",
    "# We will use not for negation (may not be perfect but decent enough)\n",
    "stops.remove('not')\n",
    "word_lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "cVille_preprocessed_df['preprocessedText'] = [feature_reduced_text(text, stop_words=stops, lemmatizer=word_lemmatizer) for text in cVille_preprocessed_df['filteredText'].tolist()]\n",
    "womenMarch_preprocessed_df['preprocessedText'] = [feature_reduced_text(text, stop_words=stops, lemmatizer=word_lemmatizer) for text in womenMarch_preprocessed_df['filteredText'].tolist()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Start Data transformation (negation handling, sentiment/polarity score and feature extraction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Negation handling (replace (isn't, ain't, haven't etc with not. stop words don't matter anyways))\n",
    "\n",
    "def getNegationHandledText(text):\n",
    "    negation_pattern = re.compile(r\"(\\w*)n't \")\n",
    "    cleanedText = re.sub(negation_pattern, 'not ', text)\n",
    "    \n",
    "    # phrase occuring after not (combine all sub phrases to one phrase)\n",
    "    negationSplit = cleanedText.split(' not ')\n",
    "    positivePhrase = negationSplit[0]\n",
    "    negativePhrase = ' '.join(negationSplit[1:])\n",
    "                       \n",
    "    # return words with _NEG appended to them\n",
    "    if negativePhrase:\n",
    "        return ' '.join([positivePhrase, ' '.join([word + \"_NEG\" for word in negativePhrase.split(' ') if word != \" \"])])\n",
    "    else:\n",
    "        return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# add negation handled text to Charlottesville df\n",
    "cVille_preprocessed_df['negationText'] = [getNegationHandledText(text) for text in cVille_preprocessed_df['preprocessedText']]\n",
    "\n",
    "# add negation handled text to Women March df\n",
    "womenMarch_preprocessed_df['negationText'] = [getNegationHandledText(text) for text in womenMarch_preprocessed_df['preprocessedText']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get sentiment score for hashtag sentences\n",
    "def hashtag_sentiment_score(hashtags):\n",
    "    # Convert hashtag to words/sentences\n",
    "    def hashtagToSentence(hashtag):\n",
    "        return \" \".join([word for word in re.split('([A-Z][a-z]+)', hashtag) if word])\n",
    "    \n",
    "    scores = [vaderAnalyzer.polarity_scores(hashtagToSentence(hashtag))['compound'] for hashtag in hashtags]\n",
    "    return scores\n",
    "\n",
    "def sentiment_score(tweet, analyzer):\n",
    "    hashtag_scores = hashtag_sentiment_score(tweet['hashtags'].split())\n",
    "    text_sentiment_score = analyzer.polarity_scores(tweet['preprocessedText'])['compound']\n",
    "    \n",
    "    normalized_score = (np.sum(hashtag_scores) + text_sentiment_score) / (len(hashtag_scores) + 1)\n",
    "    return normalized_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate sentiment score/polarity calculation. Here, we use two methods to calculate polarity scores.\n",
    "1. Vader Sentiment Intensity Analyzer that takes care of negation and is built for messy social media data like Twitter\n",
    "2. Our approach to negate the polarities of words existing after not (not itself is not considered in this case)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Part 1 - Calculate Vader score for text\n",
    "vaderAnalyzer = SentimentIntensityAnalyzer()\n",
    "\n",
    "cVille_preprocessed_df['vaderTextScore'] = [vaderAnalyzer.polarity_scores(text)['compound'] for text in cVille_preprocessed_df['preprocessedText']]\n",
    "womenMarch_preprocessed_df['vaderTextScore'] = [vaderAnalyzer.polarity_scores(text)['compound'] for text in womenMarch_preprocessed_df['preprocessedText']]\n",
    "\n",
    "# Part 2 - Calculate _NEG handled score\n",
    "\n",
    "# Calculate normalized sentiment score with Vader (both text and hashtags)\n",
    "cVille_preprocessed_df['vaderScore'] = [sentiment_score(row, vaderAnalyzer) for index, row in cVille_preprocessed_df.iterrows()]\n",
    "womenMarch_preprocessed_df['vaderScore'] = [sentiment_score(row, vaderAnalyzer) for index, row in womenMarch_preprocessed_df.iterrows()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>filteredText</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>favorites</th>\n",
       "      <th>retweets</th>\n",
       "      <th>mention_count</th>\n",
       "      <th>hashtag_count</th>\n",
       "      <th>pos_emoji</th>\n",
       "      <th>neg_emoji</th>\n",
       "      <th>punc_excl</th>\n",
       "      <th>punc_ques</th>\n",
       "      <th>violent</th>\n",
       "      <th>preprocessedText</th>\n",
       "      <th>vaderTextScore</th>\n",
       "      <th>vaderScore</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>884925177917845504</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>de blasio’s trip was approved for ‘city purpos...</td>\n",
       "      <td>#commies #antifa</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>’ trip approved ‘ city purpose ’ bashing trump...</td>\n",
       "      <td>0.4215</td>\n",
       "      <td>0.140500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>884924281951576064</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>alt-right trump-supporting neo-nazi confirmed.</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>alt-right trump-supporting neo-nazi confirmed</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>884923880938328066</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>this. kills me. i remember seeing  around duri...</td>\n",
       "      <td>#RobertGrodt #Occupy #BijiRojava #antifa</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>kill remember seeing around rest power comrade</td>\n",
       "      <td>-0.6908</td>\n",
       "      <td>-0.138160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>884923588960280576</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>is resorting to violence and unrest to silenc...</td>\n",
       "      <td>#antifa</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>resorting violence unrest silence opposition f...</td>\n",
       "      <td>-0.8271</td>\n",
       "      <td>-0.413550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>884922175748059136</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>neo-nazi  july 12th internet-wide day of act...</td>\n",
       "      <td>#Politics #Collusion #NetNeutrality</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>neo-nazi july internet-wide day action save</td>\n",
       "      <td>0.4939</td>\n",
       "      <td>0.123475</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id        date  \\\n",
       "0  884925177917845504  2017-07-11   \n",
       "1  884924281951576064  2017-07-11   \n",
       "2  884923880938328066  2017-07-11   \n",
       "3  884923588960280576  2017-07-11   \n",
       "4  884922175748059136  2017-07-11   \n",
       "\n",
       "                                        filteredText  \\\n",
       "0  de blasio’s trip was approved for ‘city purpos...   \n",
       "1     alt-right trump-supporting neo-nazi confirmed.   \n",
       "2  this. kills me. i remember seeing  around duri...   \n",
       "3   is resorting to violence and unrest to silenc...   \n",
       "4    neo-nazi  july 12th internet-wide day of act...   \n",
       "\n",
       "                                   hashtags  favorites  retweets  \\\n",
       "0                          #commies #antifa          0         2   \n",
       "1                                                    0         0   \n",
       "2  #RobertGrodt #Occupy #BijiRojava #antifa          7         4   \n",
       "3                                   #antifa          0         0   \n",
       "4       #Politics #Collusion #NetNeutrality          0         1   \n",
       "\n",
       "   mention_count  hashtag_count  pos_emoji  neg_emoji  punc_excl  punc_ques  \\\n",
       "0              0              2          0          0          0          0   \n",
       "1              0              0          0          0          0          0   \n",
       "2              0              4          0          0          0          0   \n",
       "3              0              1          0          0          0          0   \n",
       "4              1              3          0          0          0          0   \n",
       "\n",
       "   violent                                   preprocessedText  vaderTextScore  \\\n",
       "0        1  ’ trip approved ‘ city purpose ’ bashing trump...          0.4215   \n",
       "1        1      alt-right trump-supporting neo-nazi confirmed          0.0000   \n",
       "2        1     kill remember seeing around rest power comrade         -0.6908   \n",
       "3        1  resorting violence unrest silence opposition f...         -0.8271   \n",
       "4        1        neo-nazi july internet-wide day action save          0.4939   \n",
       "\n",
       "   vaderScore  \n",
       "0    0.140500  \n",
       "1    0.000000  \n",
       "2   -0.138160  \n",
       "3   -0.413550  \n",
       "4    0.123475  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cVille_preprocessed_df.drop(['negationText'], axis=1, inplace=True)\n",
    "womenMarch_preprocessed_df.drop(['negationText'], axis=1, inplace=True)\n",
    "\n",
    "cVille_preprocessed_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>filteredText</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>favorites</th>\n",
       "      <th>retweets</th>\n",
       "      <th>mention_count</th>\n",
       "      <th>hashtag_count</th>\n",
       "      <th>pos_emoji</th>\n",
       "      <th>neg_emoji</th>\n",
       "      <th>punc_excl</th>\n",
       "      <th>punc_ques</th>\n",
       "      <th>violent</th>\n",
       "      <th>preprocessedText</th>\n",
       "      <th>vaderTextScore</th>\n",
       "      <th>vaderScore</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>921443913398972416</td>\n",
       "      <td>2017-10-20</td>\n",
       "      <td>can we make our message any clearer?</td>\n",
       "      <td>#MeToo #EndRapeCulture #NotOkay #WhyIMarch</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>make message clearer</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>-0.172060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>921204515738546176</td>\n",
       "      <td>2017-10-19</td>\n",
       "      <td>anyone who doesn't support affordable  is not ...</td>\n",
       "      <td>#birthcontrol #prolife #ProofIsInThePudding #W...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>anyone n't support affordable not</td>\n",
       "      <td>-0.3089</td>\n",
       "      <td>-0.051483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>921100053720256514</td>\n",
       "      <td>2017-10-19</td>\n",
       "      <td>each and every day for</td>\n",
       "      <td>#WhyIMarch #RacialJustice #M4RJ #BlackLivesMatter</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>day</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.110500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>920843500781953024</td>\n",
       "      <td>2017-10-18</td>\n",
       "      <td>because  deserve a livable future.</td>\n",
       "      <td>#whyimarch #youth</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>deserve livable future</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>920765197605855232</td>\n",
       "      <td>2017-10-18</td>\n",
       "      <td>started a conversation with my daughter  . i ...</td>\n",
       "      <td>#MeToo #WhyIMarch #ShePersisted</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>started conversation daughter pray never exper...</td>\n",
       "      <td>0.6310</td>\n",
       "      <td>0.157750</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id        date  \\\n",
       "0  921443913398972416  2017-10-20   \n",
       "1  921204515738546176  2017-10-19   \n",
       "2  921100053720256514  2017-10-19   \n",
       "3  920843500781953024  2017-10-18   \n",
       "4  920765197605855232  2017-10-18   \n",
       "\n",
       "                                        filteredText  \\\n",
       "0             can we make our message any clearer?     \n",
       "1  anyone who doesn't support affordable  is not ...   \n",
       "2                           each and every day for     \n",
       "3                because  deserve a livable future.    \n",
       "4   started a conversation with my daughter  . i ...   \n",
       "\n",
       "                                            hashtags  favorites  retweets  \\\n",
       "0         #MeToo #EndRapeCulture #NotOkay #WhyIMarch          0         1   \n",
       "1  #birthcontrol #prolife #ProofIsInThePudding #W...          1         1   \n",
       "2  #WhyIMarch #RacialJustice #M4RJ #BlackLivesMatter          0         2   \n",
       "3                                  #whyimarch #youth          0         1   \n",
       "4                    #MeToo #WhyIMarch #ShePersisted          2         3   \n",
       "\n",
       "   mention_count  hashtag_count  pos_emoji  neg_emoji  punc_excl  punc_ques  \\\n",
       "0              0              4          0          0          0          0   \n",
       "1              0              5          0          0          0          0   \n",
       "2              0              4          0          0          0          0   \n",
       "3              1              2          0          0          0          0   \n",
       "4              1              3          0          0          0          0   \n",
       "\n",
       "   violent                                   preprocessedText  vaderTextScore  \\\n",
       "0        0                               make message clearer          0.0000   \n",
       "1        0                  anyone n't support affordable not         -0.3089   \n",
       "2        0                                                day          0.0000   \n",
       "3        0                             deserve livable future          0.0000   \n",
       "4        0  started conversation daughter pray never exper...          0.6310   \n",
       "\n",
       "   vaderScore  \n",
       "0   -0.172060  \n",
       "1   -0.051483  \n",
       "2    0.110500  \n",
       "3    0.000000  \n",
       "4    0.157750  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "womenMarch_preprocessed_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getDays(dateStrings, finalDateStr):\n",
    "    dateFormat = '%Y-%m-%d'\n",
    "    \n",
    "    def dateFromStr(dStr):\n",
    "        return datetime.strptime(dStr, dateFormat)\n",
    "    \n",
    "    finalDate = dateFromStr(finalDateStr)   \n",
    "    return [(finalDate - dateFromStr(dateStr)).days for dateStr in dateStrings]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove entries which are during the events as they dont help in predicting (discuss with the team)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get days\n",
    "cVille_preprocessed_df['days_to_event'] = getDays(cVille_preprocessed_df['date'], '2017-08-11')\n",
    "womenMarch_preprocessed_df['days_to_event'] = getDays(womenMarch_preprocessed_df['date'], '2017-01-21')\n",
    "\n",
    "# get relevant data\n",
    "cVille_preprocessed_relevant_df = cVille_preprocessed_df[(cVille_preprocessed_df.days_to_event <= 60) & (cVille_preprocessed_df.days_to_event > 0)]\n",
    "womenMarch_preprocessed_relevant_df = womenMarch_preprocessed_df[(womenMarch_preprocessed_df.days_to_event <= 60) & (womenMarch_preprocessed_df.days_to_event > 0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import collections\n",
    "# collections.Counter(womenMarch_preprocessed_relevant_df['date'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Feature Extraction\n",
    "\n",
    "1. Tf-idf vectorizer\n",
    "2. Subset relevant features in a data frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 30636 entries, 0 to 154019\n",
      "Data columns (total 15 columns):\n",
      "id                  30636 non-null int64\n",
      "date                30636 non-null object\n",
      "days_to_event       30636 non-null int64\n",
      "preprocessedText    30636 non-null object\n",
      "favorites           30636 non-null int64\n",
      "retweets            30636 non-null int64\n",
      "mention_count       30636 non-null int64\n",
      "hashtag_count       30636 non-null int64\n",
      "pos_emoji           30636 non-null int64\n",
      "neg_emoji           30636 non-null int64\n",
      "punc_excl           30636 non-null int64\n",
      "punc_ques           30636 non-null int64\n",
      "vaderTextScore      30636 non-null float64\n",
      "vaderScore          30636 non-null float64\n",
      "violent             30636 non-null int64\n",
      "dtypes: float64(2), int64(11), object(2)\n",
      "memory usage: 3.7+ MB\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 29751 entries, 40121 to 109966\n",
      "Data columns (total 15 columns):\n",
      "id                  29751 non-null int64\n",
      "date                29751 non-null object\n",
      "days_to_event       29751 non-null int64\n",
      "preprocessedText    29751 non-null object\n",
      "favorites           29751 non-null int64\n",
      "retweets            29751 non-null int64\n",
      "mention_count       29751 non-null int64\n",
      "hashtag_count       29751 non-null int64\n",
      "pos_emoji           29751 non-null int64\n",
      "neg_emoji           29751 non-null int64\n",
      "punc_excl           29751 non-null int64\n",
      "punc_ques           29751 non-null int64\n",
      "vaderTextScore      29751 non-null float64\n",
      "vaderScore          29751 non-null float64\n",
      "violent             29751 non-null int64\n",
      "dtypes: float64(2), int64(11), object(2)\n",
      "memory usage: 3.6+ MB\n"
     ]
    }
   ],
   "source": [
    "cVille_features_df = cVille_preprocessed_relevant_df[['id', 'date', 'days_to_event', 'preprocessedText', 'favorites', 'retweets', 'mention_count', 'hashtag_count', 'pos_emoji', 'neg_emoji', 'punc_excl', 'punc_ques', 'vaderTextScore', 'vaderScore', 'violent']]\n",
    "womenMarch_features_df = womenMarch_preprocessed_relevant_df[['id', 'date', 'days_to_event', 'preprocessedText', 'favorites', 'retweets', 'mention_count', 'hashtag_count', 'pos_emoji', 'neg_emoji', 'punc_excl', 'punc_ques', 'vaderTextScore', 'vaderScore', 'violent']]\n",
    "\n",
    "cVille_features_df.info()\n",
    "womenMarch_features_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>days_to_event</th>\n",
       "      <th>preprocessedText</th>\n",
       "      <th>favorites</th>\n",
       "      <th>retweets</th>\n",
       "      <th>mention_count</th>\n",
       "      <th>hashtag_count</th>\n",
       "      <th>pos_emoji</th>\n",
       "      <th>neg_emoji</th>\n",
       "      <th>punc_excl</th>\n",
       "      <th>punc_ques</th>\n",
       "      <th>vaderTextScore</th>\n",
       "      <th>vaderScore</th>\n",
       "      <th>violent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>884925177917845504</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>31</td>\n",
       "      <td>’ trip approved ‘ city purpose ’ bashing trump...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4215</td>\n",
       "      <td>0.140500</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>884924281951576064</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>31</td>\n",
       "      <td>alt-right trump-supporting neo-nazi confirmed</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>884923880938328066</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>31</td>\n",
       "      <td>kill remember seeing around rest power comrade</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.6908</td>\n",
       "      <td>-0.138160</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>884923588960280576</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>31</td>\n",
       "      <td>resorting violence unrest silence opposition f...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.8271</td>\n",
       "      <td>-0.413550</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>884922175748059136</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>31</td>\n",
       "      <td>neo-nazi july internet-wide day action save</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4939</td>\n",
       "      <td>0.123475</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id        date  days_to_event  \\\n",
       "0  884925177917845504  2017-07-11             31   \n",
       "1  884924281951576064  2017-07-11             31   \n",
       "2  884923880938328066  2017-07-11             31   \n",
       "3  884923588960280576  2017-07-11             31   \n",
       "4  884922175748059136  2017-07-11             31   \n",
       "\n",
       "                                    preprocessedText  favorites  retweets  \\\n",
       "0  ’ trip approved ‘ city purpose ’ bashing trump...          0         2   \n",
       "1      alt-right trump-supporting neo-nazi confirmed          0         0   \n",
       "2     kill remember seeing around rest power comrade          7         4   \n",
       "3  resorting violence unrest silence opposition f...          0         0   \n",
       "4        neo-nazi july internet-wide day action save          0         1   \n",
       "\n",
       "   mention_count  hashtag_count  pos_emoji  neg_emoji  punc_excl  punc_ques  \\\n",
       "0              0              2          0          0          0          0   \n",
       "1              0              0          0          0          0          0   \n",
       "2              0              4          0          0          0          0   \n",
       "3              0              1          0          0          0          0   \n",
       "4              1              3          0          0          0          0   \n",
       "\n",
       "   vaderTextScore  vaderScore  violent  \n",
       "0          0.4215    0.140500        1  \n",
       "1          0.0000    0.000000        1  \n",
       "2         -0.6908   -0.138160        1  \n",
       "3         -0.8271   -0.413550        1  \n",
       "4          0.4939    0.123475        1  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_features_df = pd.concat([cVille_features_df, womenMarch_features_df])\n",
    "training_features_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60387, 15)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_features_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# use tf-idf vectorizer\n",
    "tv = TfidfVectorizer(stop_words='english', min_df=0.0005)\n",
    "tfidf_features = tv.fit_transform(training_features_df['preprocessedText'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60387, 1855)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# tv.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create pandas dataframe from tfidf features\n",
    "tfidf_features_df = pd.DataFrame(tfidf_features.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# reset indices so that they can be concatenated into one dataframe\n",
    "temp_training_features_df = training_features_df.reset_index(drop=True)\n",
    "temp_tfidf_features_df = tfidf_features_df.reset_index(drop=True)\n",
    "\n",
    "final_training_features_df = pd.concat([temp_training_features_df, temp_tfidf_features_df], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60387, 1870)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_training_features_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>days_to_event</th>\n",
       "      <th>preprocessedText</th>\n",
       "      <th>favorites</th>\n",
       "      <th>retweets</th>\n",
       "      <th>mention_count</th>\n",
       "      <th>hashtag_count</th>\n",
       "      <th>pos_emoji</th>\n",
       "      <th>neg_emoji</th>\n",
       "      <th>...</th>\n",
       "      <th>1845</th>\n",
       "      <th>1846</th>\n",
       "      <th>1847</th>\n",
       "      <th>1848</th>\n",
       "      <th>1849</th>\n",
       "      <th>1850</th>\n",
       "      <th>1851</th>\n",
       "      <th>1852</th>\n",
       "      <th>1853</th>\n",
       "      <th>1854</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>884925177917845504</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>31</td>\n",
       "      <td>’ trip approved ‘ city purpose ’ bashing trump...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.449947</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>884924281951576064</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>31</td>\n",
       "      <td>alt-right trump-supporting neo-nazi confirmed</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>884923880938328066</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>31</td>\n",
       "      <td>kill remember seeing around rest power comrade</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>884923588960280576</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>31</td>\n",
       "      <td>resorting violence unrest silence opposition f...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>884922175748059136</td>\n",
       "      <td>2017-07-11</td>\n",
       "      <td>31</td>\n",
       "      <td>neo-nazi july internet-wide day action save</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1870 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id        date  days_to_event  \\\n",
       "0  884925177917845504  2017-07-11             31   \n",
       "1  884924281951576064  2017-07-11             31   \n",
       "2  884923880938328066  2017-07-11             31   \n",
       "3  884923588960280576  2017-07-11             31   \n",
       "4  884922175748059136  2017-07-11             31   \n",
       "\n",
       "                                    preprocessedText  favorites  retweets  \\\n",
       "0  ’ trip approved ‘ city purpose ’ bashing trump...          0         2   \n",
       "1      alt-right trump-supporting neo-nazi confirmed          0         0   \n",
       "2     kill remember seeing around rest power comrade          7         4   \n",
       "3  resorting violence unrest silence opposition f...          0         0   \n",
       "4        neo-nazi july internet-wide day action save          0         1   \n",
       "\n",
       "   mention_count  hashtag_count  pos_emoji  neg_emoji  ...   1845  1846  1847  \\\n",
       "0              0              2          0          0  ...    0.0   0.0   0.0   \n",
       "1              0              0          0          0  ...    0.0   0.0   0.0   \n",
       "2              0              4          0          0  ...    0.0   0.0   0.0   \n",
       "3              0              1          0          0  ...    0.0   0.0   0.0   \n",
       "4              1              3          0          0  ...    0.0   0.0   0.0   \n",
       "\n",
       "       1848  1849  1850  1851  1852  1853  1854  \n",
       "0  0.449947   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "1  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "2  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "3  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "4  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "\n",
       "[5 rows x 1870 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_training_features_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# drop id, date and preprocessedText from final training set\n",
    "final_training_features_df.drop(['id', 'date', 'preprocessedText'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>days_to_event</th>\n",
       "      <th>favorites</th>\n",
       "      <th>retweets</th>\n",
       "      <th>mention_count</th>\n",
       "      <th>hashtag_count</th>\n",
       "      <th>pos_emoji</th>\n",
       "      <th>neg_emoji</th>\n",
       "      <th>punc_excl</th>\n",
       "      <th>punc_ques</th>\n",
       "      <th>vaderTextScore</th>\n",
       "      <th>...</th>\n",
       "      <th>1845</th>\n",
       "      <th>1846</th>\n",
       "      <th>1847</th>\n",
       "      <th>1848</th>\n",
       "      <th>1849</th>\n",
       "      <th>1850</th>\n",
       "      <th>1851</th>\n",
       "      <th>1852</th>\n",
       "      <th>1853</th>\n",
       "      <th>1854</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4215</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.449947</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>31</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.6908</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.8271</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4939</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1867 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   days_to_event  favorites  retweets  mention_count  hashtag_count  \\\n",
       "0             31          0         2              0              2   \n",
       "1             31          0         0              0              0   \n",
       "2             31          7         4              0              4   \n",
       "3             31          0         0              0              1   \n",
       "4             31          0         1              1              3   \n",
       "\n",
       "   pos_emoji  neg_emoji  punc_excl  punc_ques  vaderTextScore  ...   1845  \\\n",
       "0          0          0          0          0          0.4215  ...    0.0   \n",
       "1          0          0          0          0          0.0000  ...    0.0   \n",
       "2          0          0          0          0         -0.6908  ...    0.0   \n",
       "3          0          0          0          0         -0.8271  ...    0.0   \n",
       "4          0          0          0          0          0.4939  ...    0.0   \n",
       "\n",
       "   1846  1847      1848  1849  1850  1851  1852  1853  1854  \n",
       "0   0.0   0.0  0.449947   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "1   0.0   0.0  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "2   0.0   0.0  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "3   0.0   0.0  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "4   0.0   0.0  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "\n",
       "[5 rows x 1867 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_training_features_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 60387 entries, 0 to 60386\n",
      "Columns: 1867 entries, days_to_event to 1854\n",
      "dtypes: float64(1857), int64(10)\n",
      "memory usage: 860.2 MB\n"
     ]
    }
   ],
   "source": [
    "final_training_features_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data Exploration and Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>days_to_event</th>\n",
       "      <th>favorites</th>\n",
       "      <th>retweets</th>\n",
       "      <th>mention_count</th>\n",
       "      <th>hashtag_count</th>\n",
       "      <th>pos_emoji</th>\n",
       "      <th>neg_emoji</th>\n",
       "      <th>punc_excl</th>\n",
       "      <th>punc_ques</th>\n",
       "      <th>vaderTextScore</th>\n",
       "      <th>...</th>\n",
       "      <th>1845</th>\n",
       "      <th>1846</th>\n",
       "      <th>1847</th>\n",
       "      <th>1848</th>\n",
       "      <th>1849</th>\n",
       "      <th>1850</th>\n",
       "      <th>1851</th>\n",
       "      <th>1852</th>\n",
       "      <th>1853</th>\n",
       "      <th>1854</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.0</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.000000</td>\n",
       "      <td>60387.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>20.636412</td>\n",
       "      <td>17.848825</td>\n",
       "      <td>8.525875</td>\n",
       "      <td>0.387252</td>\n",
       "      <td>2.133042</td>\n",
       "      <td>0.002269</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.018845</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.014516</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002826</td>\n",
       "      <td>0.000693</td>\n",
       "      <td>0.000310</td>\n",
       "      <td>0.000396</td>\n",
       "      <td>0.002770</td>\n",
       "      <td>0.000806</td>\n",
       "      <td>0.000351</td>\n",
       "      <td>0.000742</td>\n",
       "      <td>0.000250</td>\n",
       "      <td>0.000299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>19.347051</td>\n",
       "      <td>270.022484</td>\n",
       "      <td>116.562345</td>\n",
       "      <td>0.829832</td>\n",
       "      <td>1.783077</td>\n",
       "      <td>0.047577</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.135979</td>\n",
       "      <td>0.118757</td>\n",
       "      <td>0.420837</td>\n",
       "      <td>...</td>\n",
       "      <td>0.042960</td>\n",
       "      <td>0.018535</td>\n",
       "      <td>0.012761</td>\n",
       "      <td>0.014783</td>\n",
       "      <td>0.036732</td>\n",
       "      <td>0.020912</td>\n",
       "      <td>0.013863</td>\n",
       "      <td>0.018324</td>\n",
       "      <td>0.011309</td>\n",
       "      <td>0.011749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.980500</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.205700</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>14.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>35.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.318200</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>60.000000</td>\n",
       "      <td>25679.000000</td>\n",
       "      <td>9456.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.997200</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.774776</td>\n",
       "      <td>0.730130</td>\n",
       "      <td>0.952887</td>\n",
       "      <td>0.735571</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.804263</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.864720</td>\n",
       "      <td>0.697594</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 1867 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       days_to_event     favorites      retweets  mention_count  \\\n",
       "count   60387.000000  60387.000000  60387.000000   60387.000000   \n",
       "mean       20.636412     17.848825      8.525875       0.387252   \n",
       "std        19.347051    270.022484    116.562345       0.829832   \n",
       "min         1.000000      0.000000      0.000000       0.000000   \n",
       "25%         3.000000      1.000000      0.000000       0.000000   \n",
       "50%        14.000000      2.000000      1.000000       0.000000   \n",
       "75%        35.000000      5.000000      2.000000       1.000000   \n",
       "max        60.000000  25679.000000   9456.000000      14.000000   \n",
       "\n",
       "       hashtag_count     pos_emoji  neg_emoji     punc_excl     punc_ques  \\\n",
       "count   60387.000000  60387.000000    60387.0  60387.000000  60387.000000   \n",
       "mean        2.133042      0.002269        0.0      0.018845      0.014308   \n",
       "std         1.783077      0.047577        0.0      0.135979      0.118757   \n",
       "min         0.000000      0.000000        0.0      0.000000      0.000000   \n",
       "25%         1.000000      0.000000        0.0      0.000000      0.000000   \n",
       "50%         2.000000      0.000000        0.0      0.000000      0.000000   \n",
       "75%         3.000000      0.000000        0.0      0.000000      0.000000   \n",
       "max        24.000000      1.000000        0.0      1.000000      1.000000   \n",
       "\n",
       "       vaderTextScore      ...               1845          1846          1847  \\\n",
       "count    60387.000000      ...       60387.000000  60387.000000  60387.000000   \n",
       "mean         0.014516      ...           0.002826      0.000693      0.000310   \n",
       "std          0.420837      ...           0.042960      0.018535      0.012761   \n",
       "min         -0.980500      ...           0.000000      0.000000      0.000000   \n",
       "25%         -0.205700      ...           0.000000      0.000000      0.000000   \n",
       "50%          0.000000      ...           0.000000      0.000000      0.000000   \n",
       "75%          0.318200      ...           0.000000      0.000000      0.000000   \n",
       "max          0.997200      ...           1.000000      0.774776      0.730130   \n",
       "\n",
       "               1848          1849          1850          1851          1852  \\\n",
       "count  60387.000000  60387.000000  60387.000000  60387.000000  60387.000000   \n",
       "mean       0.000396      0.002770      0.000806      0.000351      0.000742   \n",
       "std        0.014783      0.036732      0.020912      0.013863      0.018324   \n",
       "min        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "max        0.952887      0.735571      1.000000      0.804263      1.000000   \n",
       "\n",
       "               1853          1854  \n",
       "count  60387.000000  60387.000000  \n",
       "mean       0.000250      0.000299  \n",
       "std        0.011309      0.011749  \n",
       "min        0.000000      0.000000  \n",
       "25%        0.000000      0.000000  \n",
       "50%        0.000000      0.000000  \n",
       "75%        0.000000      0.000000  \n",
       "max        0.864720      0.697594  \n",
       "\n",
       "[8 rows x 1867 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_training_features_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "days_to_event  violent\n",
       "1              0          0.049750\n",
       "               1         -0.025316\n",
       "2              0          0.046614\n",
       "               1         -0.044225\n",
       "3              0          0.053238\n",
       "               1         -0.037141\n",
       "4              0          0.053704\n",
       "               1         -0.044754\n",
       "5              0          0.058626\n",
       "               1         -0.038870\n",
       "6              0          0.052644\n",
       "               1         -0.071775\n",
       "7              0          0.051603\n",
       "               1         -0.042214\n",
       "8              0          0.044503\n",
       "               1         -0.034448\n",
       "9              0          0.038151\n",
       "               1         -0.039580\n",
       "10             0          0.057710\n",
       "               1         -0.051179\n",
       "11             0          0.035761\n",
       "               1         -0.071852\n",
       "12             0          0.069940\n",
       "               1         -0.055113\n",
       "13             0          0.079391\n",
       "               1         -0.069485\n",
       "14             0          0.036559\n",
       "               1         -0.068992\n",
       "15             0          0.033696\n",
       "               1         -0.044869\n",
       "                            ...   \n",
       "44             1         -0.073423\n",
       "45             0          0.037614\n",
       "               1         -0.056743\n",
       "46             0          0.087837\n",
       "               1         -0.053409\n",
       "47             0          0.017481\n",
       "               1         -0.048043\n",
       "48             0          0.031986\n",
       "               1         -0.058652\n",
       "49             0          0.060396\n",
       "               1         -0.031295\n",
       "50             0         -0.037000\n",
       "               1         -0.042323\n",
       "51             0          0.114700\n",
       "               1         -0.079688\n",
       "52             1         -0.061201\n",
       "53             0         -0.056684\n",
       "               1         -0.063693\n",
       "54             0         -0.187957\n",
       "               1         -0.079456\n",
       "55             0          0.042153\n",
       "               1         -0.051501\n",
       "56             1         -0.076121\n",
       "57             0          0.277533\n",
       "               1         -0.096656\n",
       "58             1         -0.105521\n",
       "59             0          0.048867\n",
       "               1         -0.104595\n",
       "60             0          0.000000\n",
       "               1         -0.090329\n",
       "Name: vaderScore, Length: 117, dtype: float64"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_training_features_df.groupby(['days_to_event', 'violent'])['vaderScore'].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking the pattern of Vader Sentiment Score (Text + Hashtags) in 60 days before both the events"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get subset dataframe with days_to_event, vaderScore and violent label\n",
    "vader_df = final_training_features_df[['days_to_event', 'vaderScore', 'violent']].sort_values(by='days_to_event')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# split into violent and non-violent sub-dataframe\n",
    "vader_df_vio = vader_df[vader_df.violent == 1]\n",
    "vader_df_non = vader_df[vader_df.violent == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def handleMissingDays(series):\n",
    "    series[52] = 0.0\n",
    "    series[56] = 0.0\n",
    "    series[58] = 0.0\n",
    "    return series.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# violent mean\n",
    "vio_mean = vader_df_vio.groupby('days_to_event')['vaderScore'].mean()\n",
    "\n",
    "# non-violent mean\n",
    "non_mean = vader_df_non.groupby('days_to_event')['vaderScore'].mean()\n",
    "non_mean = handleMissingDays(non_mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAG6FJREFUeJzt3Xl01OXd9/H3l7DEKkZZVBQw4G3V\nkECKScoisihLrbIcocCDLW6lFKmAS8XS47EeehTbCmg9jzenC5TyCJre3iL1ua1gqNVgJWBAIGz1\nQQmismgUEQTzff7IZJqEhCwzZpbf53VOTua3zO+6rslvPnPlmt9cY+6OiIgES4tYV0BERJqfwl9E\nJIAU/iIiAaTwFxEJIIW/iEgAKfxFRAIoKuFvZiPMbIeZ7Taz2bVsn2pmb5tZsZm9ZmYZ0ShXRESa\nxiK9zt/MUoCdwFCgFFgPTHT3bVX2OdvdPw3dHglMc/cRERUsIiJNFo2efx6w293fcfcvgeXAqKo7\nVAZ/yJmAPlkmIhJDLaNwjIuAvVWWS4Fv19zJzO4A7gJaA0PqO2iHDh08PT09CtUTEQmODRs2HHT3\njvXtF43wbxB3fxJ40sz+F/BzYHLNfcxsCjAFoGvXrhQVFTVX9UREkoKZvduQ/aIx7LMP6FJluXNo\nXV2WA6Nr2+Dui9w9x91zOnas94VLRESaKBrhvx641My6mVlrYAKwsuoOZnZplcXvAruiUK6IiDRR\nxMM+7n7SzKYDLwEpwB/cfauZPQQUuftKYLqZXQucAD6mliEfERFpPlEZ83f3F4EXa6x7oMrtGdEo\nR0REokOf8BURCSCFv4hIACn8RUQCSOEvIhJAzfYhLxGRpFbwcPXlwffHph4NpJ6/iEgAKfxFRAJI\n4S8iEkAKfxGRAFL4i4gEkMJfRCSAFP4iIgGk8BcRCSCFv4hIACn8RUQCSOEvIhJACn8RkQBS+IuI\nBJDCX0QkgBT+IiIBpPAXEQkghb+ISAAp/EVEAkjhLyISQAp/EZEAUviLiASQwl9EJIAU/iIiAaTw\nFxEJIIW/iEgAKfxFRAJI4S8iEkAKfxGRAFL4i4gEUFTC38xGmNkOM9ttZrNr2X6XmW0zs81mtsbM\nLo5GuSIi0jQRh7+ZpQBPAt8BMoCJZpZRY7e3gBx37wnkA49GWq6IiDRdNHr+ecBud3/H3b8ElgOj\nqu7g7gXufjS0+AbQOQrliohIE0Uj/C8C9lZZLg2tq8ttwP+tbYOZTTGzIjMrOnDgQBSqJiIitWnW\nN3zN7CYgB/hVbdvdfZG757h7TseOHZuzaiIigdIyCsfYB3Spstw5tK4aM7sWmAMMdPfjUShXRESa\nKBo9//XApWbWzcxaAxOAlVV3MLNvAf8JjHT3j6JQpoiIRCDi8Hf3k8B04CWgBHjG3bea2UNmNjK0\n26+As4BnzazYzFbWcTgREWkG0Rj2wd1fBF6sse6BKrevjUY5IiISHfqEr4hIACn8RUQCSOEvIhJA\nCn8RkQBS+IuIBJDCX0QkgBT+IiIBpPAXEQkghb+ISAAp/EVEAkjhLyISQAp/EZEAUviLiASQwl9E\nJIAU/iIiAaTwFxEJIIW/iEgAKfxFRAJI4S8iEkAKfxGRAFL4i4gEkMJfRCSAFP4iIgGk8BcRCSCF\nv4hIACn8RUQCSOEvIhJACn8RkQBS+IuIBJDCX0QkgFrGugIiIlFT8HD15cH3x6YeCUA9fxGRAFL4\ni4gEUFTC38xGmNkOM9ttZrNr2X61mW00s5NmNjYaZYqISNNFHP5mlgI8CXwHyAAmmllGjd3eA24G\n/k+k5YmISOSi8YZvHrDb3d8BMLPlwChgW+UO7r4ntK08CuWJiEiEojHscxGwt8pyaWidiIjEqbh6\nw9fMpphZkZkVHThwINbVERFJWtEI/31AlyrLnUPrGs3dF7l7jrvndOzYMQpVExGR2kQj/NcDl5pZ\nNzNrDUwAVkbhuCIi8jWJOPzd/SQwHXgJKAGecfetZvaQmY0EMLNcMysFxgH/aWZbIy1XRESaLirT\nO7j7i8CLNdY9UOX2eiqGg0REJA5obh8RaR6adyeuxNXVPiIi0jwU/iIiAaRhn2Sif6tFpIHU8xcR\nCSCFv4hIACn8RUQCSGP+iUrj+yISAfX8RUQCSD1/kXih/+akGSn8pfnUDDdQwInEiMJfROKf/iuK\nOoV/Q+nkE5EkovCPJxoW+Te92Ip8rRT+tYWMgkdEkpzCPxJ64RCRBJW84a8Q/no09HHV4y9Suzh5\nbiRv+NcmTh50kYjoPJYoCFb4x0oiPFkToY61iXa9E/VxEGkkhb/UrbmCMFaBm+xBn+ztay5J+jgq\n/CVySfrkOK2GvNlfuV4kDin8YyWIgSnBofM77in8E0EkTyQ9CWOvOf4G+jtLIyn8gyiIQRFvbY63\n+jSEhrWSisJfEofeGI5PuuIqIenLXEREAkjhLyISQBr2EZFTaegl6Sn8RYJEoS4hCn9JPgo4kXop\n/EXiWbK/kCV7++KYwl8k6BTAgaSrfUREAkjhLyISQFEJfzMbYWY7zGy3mc2uZXsbM1sR2v5PM0uP\nRrkiItI0EYe/maUATwLfATKAiWaWUWO324CP3f0/gPnAvEjLFRGRpotGzz8P2O3u77j7l8ByYFSN\nfUYBS0K384FrzMyiULaIiDRBNK72uQjYW2W5FPh2Xfu4+0kzKwPaAwer7mRmU4ApAF27do2oUvNP\n3lhteVYCrou3+qh9jVsXb/VJtvat+/091db1ve3Xtd/35Z3V1w39Zq33rW0dddWnlmNGe7+vW1y9\n4evui9w9x91zOnbsGOvqiIgkrWiE/z6gS5XlzqF1te5jZi2BNOBQFMoWEZEmiEb4rwcuNbNuZtYa\nmACsrLHPSmBy6PZY4BV39yiULSIiTRDxmH9oDH868BKQAvzB3bea2UNAkbuvBH4PLDWz3cBhKl4g\nREQkRqIyvYO7vwi8WGPdA1VuHwPGRaMsERGJXFy94SsiIs1D4S8iEkAKfxGRAFL4i4gEkObzl0Zp\njk8eisjXL2nDXyGVfPQ3FYmepA1/aT6xDGW9IIg0jcJfROKeXuSjT+EvdUr2J1yyt0/kdBT+zUAh\nIzXpnJBYC3z460kokpje6Dql2nLfGNUjUQU+/Bsq2i8Syf6ik+ztE0l0Cv8oU+hJTTonvh56XCMT\nqPBv6MmS7CdVsrdPROqn6R1ERAIoUD1/qZv+G5BY0HkXOwr/OKIngkj8SdbnpcJfmk2yPomiJVaP\nT3OVq79/fFH4Jyg9kSSexfv5qc8IKPxFJM7E+wtHslD4S9JReEhVOh9qp/AXkVMke2Ame/saQtf5\ni4gEkHr+SU49nAp6HESqU/gnEQWciDSUwj8Cuj46+ST7/E9fR70T9bEIOoW/xCUFisSzZDg/Ff4i\n0iDJEHjybwp/iblkCZVkaUcii+Rv0Ld7+yjWJP7pUk8RkQBSz1+kHurRSzJSz19EJIDU8xeJY/qv\nIxhi8XeOqOdvZu3M7GUz2xX6fW4d+/2PmX1iZqsiKU9ERKIj0mGf2cAad78UWBNars2vgO9HWJaI\niERJpMM+o4BBodtLgLXAfTV3cvc1Zjao5noRkWSRaEN0kfb8z3f3/aHbHwDnR3g8ERFpBvX2/M1s\nNXBBLZvmVF1wdzczj6QyZjYFmALQtWvXSA4lIiKnUW/4u/u1dW0zsw/NrJO77zezTsBHkVTG3RcB\niwBycnIieiEREZG6RTrmvxKYDDwS+v18xDUSEUkStb0PEC/vDUQ65v8IMNTMdgHXhpYxsxwz+13l\nTmb2D+BZ4BozKzWz4RGWKyIiEYio5+/uh4BrallfBNxeZXlAJOWIiEh0aXoHEZEA0vQOIkkgXsaR\nJXGo5y8iEkDq+YskGPXyJRoSKvxPnDhBaWkpx44di3VVJIGlpqbSuXNnWrVqFeuqiMRMQoV/aWkp\nbdu2JT09HTOLdXUkAbk7hw4dorS0lG7dusW6OiIxk1Bj/seOHaN9+/YKfmkyM6N9+/b671ECL6F6\n/oCCXyIWlHNI7w3I6SRUzz8efPDBB0yYMIFLLrmEK6+8kuuuu46dO3eydu1arr/++oiO/eCDD/Lr\nX//6tPssXryY999/P7y8YMECjh49GlG5NT3wwAOsXr0agEGDBlFUVARAeno6Bw8ebPBxBg0axGWX\nXUavXr3o378/O3bsaFQ9ara1MdauXUthYWGT7isSBAnX869q/ss7o3q8+npK7s6YMWOYPHkyy5cv\nB2DTpk18+OGHEZd98uTJBu23ePFiMjMzufDCC4GK8L/pppv4xje+EXEdKj300ENRO9ayZcvIyclh\n0aJF3HvvvaxcubLa9q+++oqUlJRa71uzrY2xdu1azjrrLPr169ekeoskO/X8G6GgoIBWrVoxderU\n8LpevXoxYEDF7BVHjhxh7NixXH755UyaNAn3iolJH3roIXJzc8nMzGTKlCnh9YMGDWLmzJnk5OSw\ncOHCamUVFxfTp08fevbsyZgxY/j444/Jz8+nqKiISZMmkZ2dzcKFC3n//fcZPHgwgwcPBuBvf/sb\nffv2pXfv3owbN44jR44AMHv2bDIyMujZsyf33HMPZWVlXHzxxZSXlwPw+eef06VLF06cOMHNN99M\nfn7+aR+LP//5z+Tl5ZGdnc2PfvQjvvrqq9Puf/XVV7N7926g4j+I++67j969e/Pss882qK1ffPEF\nGzZsYODAgVx55ZUMHz6c/fsrvkri8ccfD7dtwoQJ7Nmzh6eeeor58+eTnZ3NP/7xj/r/uCIBo/Bv\nhC1btnDllVfWuf2tt95iwYIFbNu2jXfeeYfXX38dgOnTp7N+/Xq2bNnCF198wapV//4q4y+//JKi\noiLuvvvuasf6wQ9+wLx589i8eTNZWVn84he/YOzYseTk5LBs2TKKi4uZMWMGF154IQUFBRQUFHDw\n4EHmzp3L6tWr2bhxIzk5OTz22GMcOnSI5557jq1bt7J582Z+/vOfk5aWRnZ2Nn//+98BWLVqFcOH\nD2/Q5Y8lJSWsWLGC119/neLiYlJSUli2bNlp7/PCCy+QlZUVXm7fvj0bN25kwoQJDWpry5Yt+clP\nfkJ+fj4bNmzg1ltvZc6ciq+UeOSRR3jrrbfYvHkzTz31FOnp6UydOpVZs2ZRXFwcfnEWkX9L6GGf\neJOXl0fnzp0ByM7OZs+ePVx11VUUFBTw6KOPcvToUQ4fPkyPHj244YYbABg/fvwpxykrK+OTTz5h\n4MCBAEyePJlx48bVW/4bb7zBtm3b6N+/P1DxwtK3b1/S0tJITU3ltttu4/rrrw+/NzF+/HhWrFjB\n4MGDWb58OdOmTWtQO9esWcOGDRvIzc0F4IsvvuC8886rdd9JkyZxxhlnkJ6ezhNPPBFeX9nuhrZ1\nx44dbNmyhaFDhwIVw0WdOnUCoGfPnkyaNInRo0czevToBrVBJOgU/o3Qo0eP0w6HtGnTJnw7JSWF\nkydPcuzYMaZNm0ZRURFdunThwQcfrHaZ4Zlnnhm1+rk7Q4cO5emnnz5l25tvvsmaNWvIz8/nt7/9\nLa+88gojR47kZz/7GYcPH2bDhg0MGTKkweVMnjyZhx9+uN59K8f8a2psu92dHj16sG7dulO2/fWv\nf+XVV1/lhRde4Je//CVvv/12o44tEkQa9mmEIUOGcPz4cRYtWhRet3nz5tOOKVcGfYcOHThy5Ei9\nY+kAaWlpnHvuueHjLl26NNwzbtu2LZ999ll436rLffr04fXXXw+PrX/++efs3LmTI0eOUFZWxnXX\nXcf8+fPZtGkTAGeddRa5ubnMmDGD66+/vs43Xmu65ppryM/P56OPKr647fDhw7z77rsNum9T23rZ\nZZdx4MCBcPifOHGCrVu3Ul5ezt69exk8eDDz5s2jrKyMI0eOnPI4iUh16vk3gpnx3HPPMXPmTObN\nm0dqairp6eksWLCAffv21Xqfc845hx/+8IdkZmZywQUXhIdK6rNkyRKmTp3K0aNH6d69O3/84x8B\nuPnmm5k6dSpnnHEG69atY8qUKYwYMSI89r948WImTpzI8ePHAZg7dy5t27Zl1KhRHDt2DHfnscce\nC5czfvx4xo0bx9q1axv8OGRkZDB37lyGDRtGeXk5rVq14sknn+Tiiy9u8DGa0tb8/HzuvPNOysrK\nOHnyJDNnzuSb3/wmN910E2VlZbg7d955J+eccw433HADY8eO5fnnn+eJJ57QuL9IDVZ55Um8ycnJ\n8crryyuVlJRwxRVXxKhGkkx0LskpCmoMYw6+Pzb1iJCZbXD3U8daa9Cwj4hIACn8RUQCSOEvIhJA\nCn8RkQBS+IuIBJDCX0QkgBT+jTBr1iwWLFgQXh4+fDi33357ePnuu++udg19rKSnp59yXXt2djaZ\nmZlROX5DJn4TkfiW2B/yqnldbqTqua63f//+PPPMM8ycOZPy8nIOHjzIp59+Gt5eWFjI/Pnzo1un\nJvrss8/Yu3cvXbp0oaSkpNH3P3nyJC1bJvbpISJ1U8+/Efr16xeeXmDr1q1kZmbStm1bPv74Y44f\nP05JSQm9e/fG3bn33nvJzMwkKyuLFStWABVzzA8cOJBRo0bRvXt3Zs+ezbJly8jLyyMrK4t//etf\nABw4cIAbb7yR3NxccnNzw7ODPvjgg9x6660MGjSI7t278/jjj9dZ1+9973vhcp9++mkmTpwY3rZn\nzx4GDBhA79696d27d/hLT9auXcuAAQMYOXIkGRkZAPzpT3+iZ8+e9OrVi+9///vhY7z66qv069eP\n7t27678AkQSkrl0jXHjhhbRs2ZL33nuPwsJC+vbty759+1i3bh1paWlkZWXRunVr/vKXv1BcXMym\nTZs4ePAgubm5XH311UDFl7+UlJTQrl07unfvzu23386bb77JwoULeeKJJ1iwYAEzZsxg1qxZXHXV\nVbz33nsMHz483Hvfvn07BQUFfPbZZ1x22WX8+Mc/rnUa5htvvJFbbrmFe+65hxdeeIFly5axdOlS\nAM477zxefvllUlNT2bVrFxMnTgx/W9fGjRvZsmUL3bp1Y+vWrcydO5fCwkI6dOjA4cOHw8ffv38/\nr732Gtu3b2fkyJGMHTv26374RSSKFP6N1K9fPwoLCyksLOSuu+5i3759FBYWkpaWFp5K+bXXXmPi\nxImkpKRw/vnnM3DgQNavX8/ZZ59Nbm5ueCriSy65hGHDhgGQlZVFQUEBAKtXr2bbtm3hMj/99NPw\nl7J897vfpU2bNrRp04bzzjuPDz/8MDyNdFXt27fn3HPPZfny5VxxxRXVvunrxIkTTJ8+PTwX/86d\n//5GtLy8PLp16wbAK6+8wrhx4+jQoQMA7dq1C+83evRoWrRoQUZGRlS+yUxEmpfCv5H69+9PYWEh\nb7/9NpmZmXTp0oXf/OY3nH322dxyyy313r/qtM8tWrQIL7do0SL8VY7l5eW88cYbpKamnvb+ldNG\n12X8+PHccccdLF68uNr6+fPnc/7557Np0ybKy8urldPQqZar1iNe54cSkbppzL+R+vXrx6pVq2jX\nrh0pKSm0a9eOTz75hHXr1oW/L3bAgAGsWLGCr776igMHDvDqq6+Sl5fX4DKGDRtW7YtPiouLm1TX\nMWPG8NOf/pThw4dXW19WVkanTp1o0aIFS5curfMrGIcMGcKzzz7LoUOHAKoN+4hIYlP4N1JWVhYH\nDx6kT58+1dalpaWFh0fGjBkTfpN0yJAhPProo1xwwQUNLuPxxx+nqKiInj17kpGRwVNPPdWkurZt\n25b77ruP1q1bV1s/bdo0lixZQq9evdi+fXudvf0ePXowZ84cBg4cSK9evbjrrruaVA8RiT+a0lkC\nSeeSnCJgUzprzF9EBBI27JtKwz4iIgEUUfibWTsze9nMdoV+n1vLPtlmts7MtprZZjMbH0mZIiIS\nuUh7/rOBNe5+KbAmtFzTUeAH7t4DGAEsMLNzmlpgvL5HIYlD55BI5OE/ClgSur0EGF1zB3ff6e67\nQrffBz4COjalsNTUVA4dOqQnrzSZu3Po0KFaP0MhEiSRvuF7vrvvD93+ADj/dDubWR7QGvhXHdun\nAFMAunbtesr2zp07U1payoEDByKpswRcampqrZ+KFgmSesPfzFYDtV2kPqfqgru7mdXZJTezTsBS\nYLK7l9e2j7svAhZBxaWeNbe3atUqPPWAiIg0Xb3h7+7X1rXNzD40s07uvj8U7h/Vsd/ZwF+BOe7+\nRpNrKyIiURHpmP9KYHLo9mTg+Zo7mFlr4DngT+6uuX9FROJApOH/CDDUzHYB14aWMbMcM/tdaJ/v\nAVcDN5tZcegnO8JyRUQkAnE7vYOZHQDebeTdOgAHv4bqxEKytCVZ2gFqS7xSW6q72N3rvaIybsO/\nKcysqCFzWiSCZGlLsrQD1JZ4pbY0jaZ3EBEJIIW/iEgAJVv4L4p1BaIoWdqSLO0AtSVeqS1NkFRj\n/iIi0jDJ1vMXEZEGSIrwN7MRZrbDzHabWW0zi8YtM/uDmX1kZluqrKt3qux4ZGZdzKzAzLaFpvCe\nEVqfcO0xs1Qze9PMNoXa8ovQ+m5m9s/QubYi9CHGuGdmKWb2lpmtCi0nZDsAzGyPmb0d+sxQUWhd\nIp5j55hZvpltN7MSM+vbnO1I+PA3sxTgSeA7QAYw0cwyYlurRllMxVTXVTVkqux4dBK4290zgD7A\nHaG/RSK25zgwxN17AdnACDPrA8wD5rv7fwAfA7fFsI6NMQMoqbKcqO2oNNjds6tcFpmI59hC4H/c\n/XKgFxV/n+Zrh7sn9A/QF3ipyvL9wP2xrlcj25AObKmyvAPoFLrdCdgR6zo2sV3PA0MTvT3AN4CN\nwLep+ABOy9D6audevP4AnUNBMgRYBVgitqNKe/YAHWqsS6hzDEgD/h+h911j0Y6E7/kDFwF7qyyX\nhtYlskZNlR2PzCwd+BbwTxK0PaGhkmIqJix8mYqpyD9x95OhXRLlXFsA/BSonE23PYnZjkoO/M3M\nNoSmgYfEO8e6AQeAP4aG435nZmfSjO1IhvBPal7RBUioS7LM7CzgL8BMd/+06rZEao+7f+Xu2VT0\nnPOAy2NcpUYzs+uBj9x9Q6zrEkVXuXtvKoZ67zCzq6tuTJBzrCXQG/jf7v4t4HNqDPF83e1IhvDf\nB3Spstw5tC6RfRiaIrvyexBqnSo7HplZKyqCf5m7/1dodcK2B8DdPwEKqBgeOcfMKqdCT4RzrT8w\n0sz2AMupGPpZSOK1I8zd94V+f0TFjMF5JN45VgqUuvs/Q8v5VLwYNFs7kiH81wOXhq5eaA1MoGKq\n6URW71TZ8cjMDPg9UOLuj1XZlHDtMbOOld81bWZnUPHeRQkVLwJjQ7vFfVvc/X537+zu6VQ8N15x\n90kkWDsqmdmZZta28jYwDNhCgp1j7v4BsNfMLgutugbYRnO2I9ZvfETpzZPrgJ1UjMnOiXV9Gln3\np4H9wAkqegO3UTEmuwbYBawG2sW6ng1sy1VU/Ju6GSgO/VyXiO0BegJvhdqyBXggtL478CawG3gW\naBPrujaiTYOAVYncjlC9N4V+tlY+3xP0HMsGikLn2H8D5zZnO/QJXxGRAEqGYR8REWkkhb+ISAAp\n/EVEAkjhLyISQAp/EZEAUviLiASQwl9EJIAU/iIiAfT/ASwF3vHNXWBoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13a142550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# draw plot\n",
    "plt.bar(range(1,61), vio_mean, alpha=0.5, label='Charlottesville Protest')\n",
    "plt.bar(range(1,61), non_mean, alpha=0.5, label='Women March')\n",
    "plt.legend(loc='lower left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that Charlotessvile protest tweets were negative (on average) and Women March tweets were positive (on average) for 60 days before the events."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.862862056328\n",
      "0.0274773961666\n",
      "0.00251803000083\n"
     ]
    }
   ],
   "source": [
    "# calculate correlation between vaderScore and vaderTextScore\n",
    "setiment_corr = final_training_features_df['vaderScore'].corr(final_training_features_df['vaderTextScore'])\n",
    "print(setiment_corr)\n",
    "\n",
    "# calculate correlation between vaderScore and presence of exclamation marks\n",
    "excl_corr = final_training_features_df['vaderScore'].corr(final_training_features_df['punc_excl'])\n",
    "print(excl_corr)\n",
    "\n",
    "# calculate correlation between vaderScore and presence of question marks\n",
    "ques_corr = final_training_features_df['vaderScore'].corr(final_training_features_df['punc_ques'])\n",
    "print(ques_corr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As expected, Sentiment Score for tweet and hashtags is highly correlated with Sentiment Score of tweet text alone.\n",
    "However, there is no real correlation between sentiment score and presence of exclamation marks or question marks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pos_emoji\n",
       "0    60250\n",
       "1      137\n",
       "Name: pos_emoji, dtype: int64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_training_features_df.groupby('pos_emoji')['pos_emoji'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "neg_emoji\n",
       "0    60387\n",
       "Name: neg_emoji, dtype: int64"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_training_features_df.groupby('neg_emoji')['neg_emoji'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "pos_emoji and neg_emoji do not give any valuable information so we will not use it for our final models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "exploratory_df = final_training_features_df[['days_to_event', 'favorites', 'mention_count', 'retweets', 'hashtag_count', 'violent']].sort_values(by='days_to_event')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "exploratory_df_vio = exploratory_df[exploratory_df.violent == 1]\n",
    "exploratory_df_non = exploratory_df[exploratory_df.violent == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGyNJREFUeJzt3Xt0VPW99/H3l4DECnIXUZBAW1Eg\nEDHJAZSr5XIsclmCStGi0lIqVkBRsXb10S66Ki4roO06Piy1oOURajw+Kj3rHBFDqQaVAAGB4PVB\nBRHCLQUVSsj3+WOGaRISMrnOzJ7Pa62szN6zZ+b3S/Z85jff+e095u6IiEjiaxLrBoiISP1QoIuI\nBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAaNqYD9a+fXtPS0trzIcUEUl4\nGzduPODuHarbrlEDPS0tjfz8/MZ8SBGRhGdmn0WznUouIiIBoUAXEQkIBbqISEA0ag29MidPnmT3\n7t0cP3481k2RBJWamkrnzp1p1qxZrJsiElMxD/Tdu3fTsmVL0tLSMLNYN0cSjLtz8OBBdu/eTbdu\n3WLdHJGYinnJ5fjx47Rr105hLrViZrRr107v8ESIg0AHFOZSJ9p/RELiItBFRKTuYl5Dr2jh6g/r\n9f7mjLi02m2++uorZs+ezYYNG2jdujUdO3Zk0aJFfPnllzz22GOsWrWq1o//0EMP0aJFC+bOnVvl\nNkuXLmXkyJFcdNFFACxatIjp06fzne98p9aPW9Gvf/1rBg8ezA9+8AOGDh3KY489RmZmZuRgr/bt\n20d1P0OHDmXv3r2kpqbSokULnn32WXr06BF1Oyr2tSbWrl3LOeecw8CBA2t8W5FkEHeB3tjcnQkT\nJjB16lRWrFgBwJYtW9i3b1+d77ukpCSq7ZYuXUrv3r3LBfrNN99cr4H+m9/8pt7ua/ny5WRmZrJk\nyRLuvfdeXn311XLXnzp1ipSUlEpvW7GvNbF27VpatGihQA+q3N+duW7YA43fjgSW9CWX3NxcmjVr\nxowZMyLr+vbty6BBgwA4duwYEydO5LLLLmPKlCm4OxAKyKysLHr37s306dMj64cOHcrs2bPJzMxk\n8eLF5R6roKCA/v3706dPHyZMmMDhw4fJyckhPz+fKVOmkJGRweLFi/nyyy8ZNmwYw4YNA+D1119n\nwIAB9OvXj0mTJnHs2DEA5s2bR8+ePenTpw9z586luLiYrl27UlpaCsDXX39Nly5dOHnyJLfeeis5\nOTln/Vv8+c9/Jjs7m4yMDH72s59x6tSps24/ePBgPv74YyB0Wof777+ffv368eKLL0bV12+//ZaN\nGzcyZMgQrrzySkaNGsXevXsBeOKJJyJ9u+mmm9i1axdPPfUUCxcuJCMjg7///e/V/3NFkkzSB/q2\nbdu48sorq7x+8+bNLFq0iB07dvDpp5/y9ttvA3DnnXeyYcMGtm3bxrfffluuLPPPf/6T/Px87rnn\nnnL39eMf/5gFCxawdetW0tPTefjhh5k4cSKZmZksX76cgoICZs2axUUXXURubi65ubkcOHCA+fPn\n88Ybb7Bp0yYyMzN5/PHHOXjwIC+//DLbt29n69at/OpXv6JVq1ZkZGTwt7/9DYBVq1YxatSoqOZn\nFxYWsnLlSt5++20KCgpISUlh+fLlZ73Na6+9Rnp6emS5Xbt2bNq0iZtuuimqvjZt2pRf/OIX5OTk\nsHHjRm6//XYefPBBAB555BE2b97M1q1beeqpp0hLS2PGjBnMmTOHgoKCyAuuiPxL0pdcqpOdnU3n\nzp0ByMjIYNeuXVx99dXk5uby6KOP8s0333Do0CF69erFddddB8CNN954xv0UFxdz5MgRhgwZAsDU\nqVOZNGlStY//zjvvsGPHDq666iog9GIxYMAAWrVqRWpqKtOmTWPMmDGMGTMm8tgrV65k2LBhrFix\ngjvuuCOqfq5Zs4aNGzeSlZUFwLfffssFF1xQ6bZTpkzh3HPPJS0tjSeffDKy/nS/o+3rBx98wLZt\n2xgxYgQQKtV06tQJgD59+jBlyhTGjx/P+PHjo+qDSLJL+kDv1avXWUsRzZs3j1xOSUmhpKSE48eP\nc8cdd5Cfn0+XLl146KGHys2DPu+88+qtfe7OiBEjeOGFF8647r333mPNmjXk5OTwhz/8gTfffJOx\nY8fyy1/+kkOHDrFx40aGDx8e9eNMnTqV3/2ukjpmBadr6BXVtN/uTq9evVi/fv0Z1/31r39l3bp1\nvPbaa/z2t7/l/fffr9F9iySjpC+5DB8+nBMnTrBkyZLIuq1bt561Rns6vNu3b8+xY8eqrU0DtGrV\nijZt2kTu9/nnn4+MYFu2bMnRo0cj25Zd7t+/P2+//XakVv3111/z4YcfcuzYMYqLi7n22mtZuHAh\nW7ZsAaBFixZkZWUxa9YsxowZU+WHkxVdc8015OTksH//fgAOHTrEZ59FdcbOWve1R48eFBUVRQL9\n5MmTbN++ndLSUr744guGDRvGggULKC4u5tixY2f8nUSkvLgboUczzbA+mRkvv/wys2fPZsGCBaSm\nppKWlsaiRYvYs2dPpbdp3bo1P/3pT+nduzcXXnhhpExRnWXLljFjxgy++eYbunfvzp/+9CcAbr31\nVmbMmMG5557L+vXrmT59OqNHj47U0pcuXcrkyZM5ceIEAPPnz6dly5aMGzeO48eP4+48/vjjkce5\n8cYbmTRpEmvXro3679CzZ0/mz5/PyJEjKS0tpVmzZvzxj3+ka9euUd9Hbfqak5PDXXfdRXFxMSUl\nJcyePZtLL72Um2++meLiYtydu+66i9atW3PdddcxceJEXnnlFZ588knV0UUqsNOzMxpDZmamV/yC\ni8LCQi6//PJGa4MEk/ajANC0xSqZ2UZ3P7POWUHSl1xERIJCgS4iEhAKdBGRgFCgi4gEhAJdRCQg\nFOgiIgERd/PQK526VBfVTHuaM2cOXbt2Zfbs2QCMGjWKLl268PTTTwNwzz33cPHFF3P33XfXb7tq\nKC0tjS5dupQ74CkjI4OSkhK2bdtW5/u/9dZbGTNmDBMnTqzzfYlIbCT9CP2qq64iLy8PgNLSUg4c\nOMD27dsj1+fl5cXN6VqPHj3KF198AYTmXddUtKfzFZHEFFWgm9kuM3vfzArMLD+8rq2ZrTazj8K/\n2zRsUxvGwIEDI4eeb9++nd69e9OyZUsOHz7MiRMnKCwspF+/frg79957L7179yY9PZ2VK1cCoXN0\nDxkyhHHjxtG9e3fmzZvH8uXLyc7OJj09nU8++QSAoqIirr/+erKyssjKyoqctfGhhx7i9ttvZ+jQ\noXTv3p0nnniiyrbecMMNkcd94YUXmDx5cuS6Xbt2MWjQIPr160e/fv0iL1Jr165l0KBBjB07lp49\newLw3HPP0adPH/r27cstt9wSuY9169YxcOBAunfvHtXpDEQkvtSk5DLM3Q+UWZ4HrHH3R8xsXnj5\n/nptXSO46KKLaNq0KZ9//jl5eXkMGDCAPXv2sH79elq1akV6ejrnnHMOL730EgUFBWzZsoUDBw6Q\nlZXF4MGDgdAXYhQWFtK2bVu6d+/OT37yE9577z0WL17Mk08+yaJFi5g1axZz5szh6quv5vPPP2fU\nqFGRUfbOnTvJzc3l6NGj9OjRg5///OeVnvL2+uuv57bbbmPu3Lm89tprLF++nOeffx6ACy64gNWr\nV5OamspHH33E5MmTOX1U7qZNm9i2bRvdunVj+/btzJ8/n7y8PNq3b8+hQ4ci9793717eeustdu7c\nydixY1V+EUkwdamhjwOGhi8vA9aSgIEOoVF6Xl4eeXl53H333ezZs4e8vDxatWoVOW3tW2+9xeTJ\nk0lJSaFjx44MGTKEDRs2cP7555OVlRU57et3v/tdRo4cCUB6ejq5ubkAvPHGG+zYsSPymP/4xz8i\nX1Txwx/+kObNm9O8eXMuuOAC9u3bFzllb1nt2rWjTZs2rFixgssvv7zcNxqdPHmSO++8M3Iu8w8/\n/NdX+WVnZ9OtWzcA3nzzTSZNmhT5yrm2bdtGths/fjxNmjShZ8+e9fKNTSLSuKINdAdeNzMH/re7\nLwE6uvve8PVfAR0ru6GZTQemA1xyySV1bG7DOF1Hf//99+nduzddunTh97//Peeffz633XZbtbcv\ne4rdJk2aRJabNGkSqVuXlpbyzjvvkJqaetbbnz5Fb1VuvPFGZs6cydKlS8utX7hwIR07dmTLli2U\nlpaWe5xoT2tbth2NeY4fEakf0X4oerW79wP+HZhpZoPLXumhZ3+lCeDuS9w9090zO3ToULfWNpCB\nAweyatUq2rZtS0pKCm3btuXIkSOsX78+8oHooEGDWLlyJadOnaKoqIh169aRnZ0d9WOMHDmy3JdB\nFBQU1KqtEyZM4L777mPUqFHl1hcXF9OpUyeaNGnC888/X+XXxw0fPpwXX3yRgwcPApQruYhIYotq\nhO7ue8K/95vZy0A2sM/MOrn7XjPrBOyvlxbF4Oxq6enpHDhwgB/96Efl1h07dixSmpgwYQLr16+n\nb9++mBmPPvooF154ITt37ozqMZ544glmzpxJnz59KCkpYfDgwTz11FM1bmvLli25//4zK1t33HEH\n119/Pc899xyjR4+uclTeq1cvHnzwQYYMGUJKSgpXXHHFGaN9EUlM1Z4+18zOA5q4+9Hw5dXAb4Br\ngINlPhRt6+73ne2+dPpcaSjajwJAp8+tUrSnz41mhN4ReNnMTm//f9z9v81sA/AXM5sGfAbcUJcG\ni4hI3VQb6O7+KdC3kvUHCY3SRUQkDsTFkaKaUSF1of1HJCTmgZ6amsrBgwf1pJRacXcOHjxY6XRQ\nkWQT85Nzde7cmd27d1NUVBTrpkiCSk1NrfRALJFkE/NAb9asWeQoRhERqb2Yl1xERKR+KNBFRAJC\ngS4iEhAKdBGRgFCgi4gERMxnuUSt4nkedI4HEZFyNEIXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGA\nUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgi\nIgGhQBcRCQgFuohIQEQd6GaWYmabzWxVeLmbmb1rZh+b2UozO6fhmikiItWpyQh9FlBYZnkBsNDd\nvwccBqbVZ8NERKRmogp0M+sM/BB4OrxswHAgJ7zJMmB8QzRQRESiE+0IfRFwH1AaXm4HHHH3kvDy\nbuDiem6biIjUQLWBbmZjgP3uvrE2D2Bm080s38zyi4qKanMXIiIShWhG6FcBY81sF7CCUKllMdDa\nzJqGt+kM7Knsxu6+xN0z3T2zQ4cO9dBkERGpTLWB7u4PuHtnd08DbgLedPcpQC4wMbzZVOCVBmul\niIhUqy7z0O8H7jazjwnV1J+pnyaJiEhtNK1+k39x97XA2vDlT4Hs+m+SiIjUho4UFREJCAW6iEhA\nKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQR\nkYBQoIuIBIQCXUQkIBToIiIBUaMvuBAJsoWrPyy3PGfEpTFqiUjtaIQuIhIQCnQRkYBQoIuIBIQC\nXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCCqDXQzSzWz98xs\ni5ltN7OHw+u7mdm7Zvaxma00s3MavrkiIlKVaEboJ4Dh7t4XyABGm1l/YAGw0N2/BxwGpjVcM0VE\npDrVBrqHHAsvNgv/ODAcyAmvXwaMb5AWiohIVKKqoZtZipkVAPuB1cAnwBF3Lwlvshu4uIrbTjez\nfDPLLyoqqo82i4hIJaIKdHc/5e4ZQGcgG7gs2gdw9yXununumR06dKhlM0VEpDo1+sYidz9iZrnA\nAKC1mTUNj9I7A3saooFSS7m/K7887IHYtENEGk00s1w6mFnr8OVzgRFAIZALTAxvNhV4paEaKSIi\n1YtmhN4JWGZmKYReAP7i7qvMbAewwszmA5uBZxqwnSIiUo1qA93dtwJXVLL+U0L1dBERiQM6UlRE\nJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhA1OlJUGs7C1R+WW54z4tIYtUREEpVG6CIiAaFA\nFxEJCJVcaksnvxKROKMRuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBERiz3LRTBOJYzpYTBqbRugi\nIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBI7GmL9aziNDPQVDMRSRwKdJF6oDnnEg9UchER\nCQgFuohIQCjQRUQCQoEuIhIQ1Qa6mXUxs1wz22Fm281sVnh9WzNbbWYfhX+3afjmiohIVaIZoZcA\n97h7T6A/MNPMegLzgDXu/n1gTXhZRERipNpAd/e97r4pfPkoUAhcDIwDloU3WwaMb6hGiohI9Wo0\nD93M0oArgHeBju6+N3zVV0DHem1ZQ9O51EUkYKL+UNTMWgAvAbPd/R9lr3N3B7yK2003s3wzyy8q\nKqpTY0VEpGpRBbqZNSMU5svd/T/Dq/eZWafw9Z2A/ZXd1t2XuHumu2d26NChPtosIiKVqLbkYmYG\nPAMUuvvjZa56FZgKPBL+/UqDtDCZqSwkIjUQTQ39KuAW4H0zKwiv+yWhIP+LmU0DPgNuaJgmiohI\nNKoNdHd/C7Aqrr6mfpsjIoGjd5qNRkeKiogEhAJdRCQgdD70INBbWhFBI3QRkcDQCL0+aaQsDUhf\nkSjVUaCLnIW+Wq5m9PeKLZVcREQCQoEuIhIQKrmIiEQp3ktKGqGLiASERuhl9P98SSVrH2v0dkgF\nmj0kEpXgBbqe/CKSpIIX6CINLN7rqJK8VEMXEQkIBbqISEAkdMll/acHyy0PGBajhiQLfT4hEtcS\nOtAlcajuHEB6gY87CnSpnJ6sIglHgR6FxhhdnjkHXvPfJblULKGCyqg1lRyBrtGmiCSB5Ah0SWqq\n30uy0LRFEZGA0Ahd6l9jlLhURhM5gwI9wTRI+UDhKBIICvRYUICKSANQoEtc0QeYIrWnQE8wmq8u\nIlVRoEtcCfoLlt6BSENKmECP9kRclW0X05N4JWi9PBFPfFYxLCFBAjNB9xGJP9UGupk9C4wB9rt7\n7/C6tsBKIA3YBdzg7ocbrpkiwdAg70D0giBh0RxYtBQYXWHdPGCNu38fWBNeFhGRGKp2hO7u68ws\nrcLqccDQ8OVlwFrg/npsl5CYZQ8RiZ3a1tA7uvve8OWvgI5VbWhm04HpAJdcckktH04SiV6IAqiS\nso7+z/Gnzh+KurubmZ/l+iXAEoDMzMwqt5NGUPFJCTV6YuoJLBLfahvo+8ysk7vvNbNOwP76bJQk\nh2in8NX1hSQRp0JqemPjCsrfu7aB/iowFXgk/PuVemtRHErEQBCR5BPNtMUXCH0A2t7MdgP/i1CQ\n/8XMpgGfATc0ZCODRqULEWkI0cxymVzFVdfUc1tE6uTMd1Kgd1OSTBLmSFERkcaUiHV1Bbokpbo8\nWePtMxWV8OQ0BXot6Ukk8SwRR5d1plMgJG+gK5ATWzJOZQwUhW+DSNpAbwjJ+CKRlCPBgKvsf1rf\n+3aQnivx9BxQoDewxthxg/TkkJBGewehkXKgKNBFJLk0xotYjF4oozl9roiIJACN0ANKZRhJNpXt\n8/FU324MCnSRs2iMWrZefKW+KNClTqINPE0TbFz6eycnBbpIA1GoSmNToItIUglyiUuBLiINKsgB\nGm8U6CL1QOWVxhXvs1di9SKmQJeYUQjWTLKe7z3ewzue6MAiEZGA0AhdJImpvh0sCvQkUvHJC3oC\nJ5P1z8wttzxgWvDLNclGgS4i5eizjcSlGrqISEBohC4SppFp3elvGFsKdBGRSiTieYpUchERCQgF\nuohIQCjQRUQCQoEuIhIQCnQRkYCo0ywXMxsNLAZSgKfd/ZF6aZWISFg8zSKJd7UeoZtZCvBH4N+B\nnsBkM+tZXw0TEZGaqUvJJRv42N0/dfd/AiuAcfXTLBERqam6BPrFwBdllneH14mISAyYu9fuhmYT\ngdHu/pPw8i3Av7n7nRW2mw5MDy/2AD6owcO0Bw7UqoHxR32JT+pL/AlKP6D++tLV3TtUt1FdPhTd\nA3Qps9w5vK4cd18CVPZVK9Uys3x3z6xd8+KL+hKf1Jf4E5R+QOP3pS4llw3A982sm5mdA9wEvFo/\nzRIRkZqq9Qjd3UvM7E7gfwhNW3zW3bfXW8tERKRG6jQP3d3/C/ivempLZWpVqolT6kt8Ul/iT1D6\nAY3cl1p/KCoiIvFFh/6LiAREXAa6mY02sw/M7GMzmxfr9tSUmT1rZvvNbFuZdW3NbLWZfRT+3SaW\nbYyGmXUxs1wz22Fm281sVnh9IvYl1czeM7Mt4b48HF7fzczeDe9rK8Mf8CcEM0sxs81mtiq8nJB9\nMbNdZva+mRWYWX54XcLtYwBm1trMcsxsp5kVmtmAxuxL3AV6QE4psBQYXWHdPGCNu38fWBNejncl\nwD3u3hPoD8wM/y8SsS8ngOHu3hfIAEabWX9gAbDQ3b8HHAamxbCNNTULKCyznMh9GebuGWWm+CXi\nPgahc1v9t7tfBvQl9P9pvL64e1z9AAOA/ymz/ADwQKzbVYt+pAHbyix/AHQKX+4EfBDrNtaiT68A\nIxK9L8B3gE3AvxE66KNpeH25fS+efwgd97EGGA6sAiyB+7ILaF9hXcLtY0Ar4P8R/mwyFn2JuxE6\nwT2lQEd33xu+/BXQMZaNqSkzSwOuAN4lQfsSLlEUAPuB1cAnwBF3Lwlvkkj72iLgPqA0vNyOxO2L\nA6+b2cbwkeWQmPtYN6AI+FO4FPa0mZ1HI/YlHgM98Dz0Up0w04vMrAXwEjDb3f9R9rpE6ou7n3L3\nDEKj22zgshg3qVbMbAyw3903xrot9eRqd+9HqMw608wGl70ygfaxpkA/4D/c/QrgayqUVxq6L/EY\n6FGdUiAB7TOzTgDh3/tj3J6omFkzQmG+3N3/M7w6IftymrsfAXIJlSVam9np4zESZV+7ChhrZrsI\nneV0OKHabSL2BXffE/69H3iZ0IttIu5ju4Hd7v5ueDmHUMA3Wl/iMdCDekqBV4Gp4ctTCdWj45qZ\nGfAMUOjuj5e5KhH70sHMWocvn0vos4BCQsE+MbxZQvTF3R9w987unkbo+fGmu08hAftiZueZWcvT\nl4GRwDYScB9z96+AL8ysR3jVNcAOGrMvsf4goYoPF64FPiRU43ww1u2pRftfAPYCJwm9ak8jVONc\nA3wEvAG0jXU7o+jH1YTeHm4FCsI/1yZoX/oAm8N92Qb8Ory+O/Ae8DHwItA81m2tYb+GAqsStS/h\nNm8J/2w//XxPxH0s3O4MID+8n/1foE1j9kVHioqIBEQ8llxERKQWFOgiIgGhQBcRCQgFuohIQCjQ\nRUQCQoEuIhIQCnQRkYBQoIuIBMT/B93RfeYazA1wAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13a0a3e48>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fav_vio_mean = exploratory_df_vio.groupby('days_to_event')['favorites'].mean()\n",
    "fav_non_mean = exploratory_df_non.groupby('days_to_event')['favorites'].mean()\n",
    "fav_non_mean = handleMissingDays(fav_non_mean)\n",
    "\n",
    "plt.bar(range(1,61), fav_vio_mean, alpha=0.5, label='Charlottesville Protest')\n",
    "plt.bar(range(1,61), fav_non_mean, alpha=0.5, label='Women March')\n",
    "plt.legend(loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Clearly, non-violent Women March tweets were favorited more on average as compared to Charlottesville tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGwBJREFUeJzt3Xt0VOW9//H3l4DECnIHUSiBtqJA\nIGKSAypXy+VnkcsSVIoWlZZStQKKFWtXf9pFV8VlBbRdx8VSC1p+Qo3Ho9KzzhExlGpQSSBgQvB6\nUIMI4RaJCiXk+f0xmzEJiZkkk8zMk89rrayZfZnZ3yfZ+cwzz+y9x5xziIhI4msV6wJERCQ6FOgi\nIp5QoIuIeEKBLiLiCQW6iIgnFOgiIp5QoIuIeEKBLiLiCQW6iIgnWjfnxrp27epSUlKac5MiIgkv\nLy/voHOuW13rNWugp6SkkJub25ybFBFJeGb2cSTrachFRMQTCnQREU8o0EVEPNGsY+g1OXnyJMXF\nxRw/fjzWpUiCSk5OplevXrRp0ybWpYjEVMwDvbi4mPbt25OSkoKZxbocSTDOOQ4dOkRxcTF9+/aN\ndTkiMRXzIZfjx4/TpUsXhbk0iJnRpUsXvcMTIQ4CHVCYS6No/xEJiYtAFxGRxov5GHp1yza8F9Xn\nWzjuwjrX+fzzz1mwYAFbt26lY8eO9OjRg+XLl/PZZ5/x8MMPs379+gZv//7776ddu3YsWrSo1nVW\nrVrF+PHjOf/88wFYvnw5c+fO5Tvf+U6Dt1vdb3/7W0aOHMkPf/hDRo8ezcMPP0x6enr4ZK+uXbtG\n9DyjR49m3759JCcn065dO5566in69+8fcR3V21ofmzZt4qyzzuKyyy6r92NFWoK4C/Tm5pxj2rRp\nzJ49m7Vr1wKwY8cO9u/f3+jnLi8vj2i9VatWMWjQoCqBfsMNN0Q10H/3u99F7bnWrFlDeno6K1eu\n5O677+all16qsvzUqVMkJSXV+Njqba2PTZs20a5dOwW6r7L/cOa8Mfc2fx0JrMUPuWRnZ9OmTRvm\nzZsXnjdkyBBGjBgBQFlZGdOnT+eiiy5i1qxZOOeAUEBmZGQwaNAg5s6dG54/evRoFixYQHp6OitW\nrKiyrfz8fIYNG8bgwYOZNm0aR44cISsri9zcXGbNmkVaWhorVqzgs88+Y8yYMYwZMwaAV155heHD\nhzN06FBmzJhBWVkZAIsXL2bAgAEMHjyYRYsWUVpaSp8+faioqADgyy+/pHfv3pw8eZKbbrqJrKys\nb/1d/PWvfyUzM5O0tDR+/vOfc+rUqW9df+TIkXzwwQdA6LIO99xzD0OHDuW5556LqK1ff/01eXl5\njBo1iksvvZQJEyawb98+AB599NFw266//nr27NnD448/zrJly0hLS+Of//xn3X9ckRamxQd6QUEB\nl156aa3Lt2/fzvLly9m1axcfffQRb7zxBgC33347W7dupaCggK+//rrKsMy//vUvcnNzueuuu6o8\n109+8hOWLl3Kzp07SU1N5YEHHmD69Omkp6ezZs0a8vPzmT9/Pueffz7Z2dlkZ2dz8OBBlixZwquv\nvsq2bdtIT0/nkUce4dChQ7zwwgsUFhayc+dOfvOb39ChQwfS0tL4xz/+AcD69euZMGFCRMdnFxUV\nsW7dOt544w3y8/NJSkpizZo13/qYl19+mdTU1PB0ly5d2LZtG9dff31EbW3dujW//OUvycrKIi8v\nj1tuuYX77rsPgAcffJDt27ezc+dOHn/8cVJSUpg3bx4LFy4kPz8//IIrIt9o8UMudcnMzKRXr14A\npKWlsWfPHq644gqys7N56KGH+Oqrrzh8+DADBw7k6quvBuC6664743lKS0s5evQoo0aNAmD27NnM\nmDGjzu2/+eab7Nq1i8svvxwIvVgMHz6cDh06kJyczJw5c5g0aRKTJk0Kb3vdunWMGTOGtWvXcuut\nt0bUzo0bN5KXl0dGRgYAX3/9Nd27d69x3VmzZnH22WeTkpLCY489Fp5/ut2RtvXdd9+loKCAcePG\nAaGhmp49ewIwePBgZs2axdSpU5k6dWpEbRBp6Vp8oA8cOPBbhyLatm0bvp+UlER5eTnHjx/n1ltv\nJTc3l969e3P//fdXOQ76nHPOiVp9zjnGjRvHs88+e8ayt99+m40bN5KVlcWf/vQnXnvtNSZPnsyv\nf/1rDh8+TF5eHmPHjo14O7Nnz+YPf6hhHLOa02Po1dW33c45Bg4cyJYtW85Y9ve//53Nmzfz8ssv\n8/vf/5533nmnXs8t0hK1+CGXsWPHcuLECVauXBmet3Pnzm8doz0d3l27dqWsrKzOsWmADh060KlT\np/DzPvPMM+EebPv27Tl27Fh43crTw4YN44033giPVX/55Ze89957lJWVUVpaylVXXcWyZcvYsWMH\nAO3atSMjI4P58+czadKkWj+crO7KK68kKyuLAwcOAHD48GE+/jiiK3Y2uK39+/enpKQkHOgnT56k\nsLCQiooKPv30U8aMGcPSpUspLS2lrKzsjN+TiFQVdz30SA4zjCYz44UXXmDBggUsXbqU5ORkUlJS\nWL58OXv37q3xMR07duRnP/sZgwYN4rzzzgsPU9Rl9erVzJs3j6+++op+/frxl7/8BYCbbrqJefPm\ncfbZZ7Nlyxbmzp3LxIkTw2Ppq1atYubMmZw4cQKAJUuW0L59e6ZMmcLx48dxzvHII4+Et3Pdddcx\nY8YMNm3aFPHvYcCAASxZsoTx48dTUVFBmzZt+POf/0yfPn0ifo6GtDUrK4s77riD0tJSysvLWbBg\nARdeeCE33HADpaWlOOe444476NixI1dffTXTp0/nxRdf5LHHHtM4ukg1dvrojOaQnp7uqn/BRVFR\nERdffHGz1SB+0n7kAR22WCszy3POnTnOWU2LH3IREfGFAl1ExBMKdBERTyjQRUQ8oUAXEfGEAl1E\nxBNxdxx6jYcuNUYdhz0tXLiQPn36sGDBAgAmTJhA7969eeKJJwC46667uOCCC7jzzjujW1c9paSk\n0Lt37yonPKWlpVFeXk5BQUGjn/+mm25i0qRJTJ8+vdHPJSKx0eJ76Jdffjk5OTkAVFRUcPDgQQoL\nC8PLc3Jy4uZyrceOHePTTz8FQsdd11ekl/MVkcTU4gP9sssuC596XlhYyKBBg2jfvj1HjhzhxIkT\nFBUVMXToUJxz3H333QwaNIjU1FTWrVsHhK7RPWrUKKZMmUK/fv1YvHgxa9asITMzk9TUVD788EMA\nSkpKuOaaa8jIyCAjIyN81cb777+fW265hdGjR9OvXz8effTRWmu99tprw9t99tlnmTlzZnjZnj17\nGDFiBEOHDmXo0KHhF6lNmzYxYsQIJk+ezIABAwB4+umnGTx4MEOGDOHGG28MP8fmzZu57LLL6Nev\nX0SXMxCR+BJ/Qy7N7Pzzz6d169Z88skn5OTkMHz4cPbu3cuWLVvo0KEDqampnHXWWTz//PPk5+ez\nY8cODh48SEZGBiNHjgRCX4hRVFRE586d6devHz/96U95++23WbFiBY899hjLly9n/vz5LFy4kCuu\nuIJPPvmECRMmhHvZu3fvJjs7m2PHjtG/f39+8Ytf1HjJ22uuuYabb76ZRYsW8fLLL7NmzRqeeeYZ\nALp3786GDRtITk7m/fffZ+bMmZw+K3fbtm0UFBTQt29fCgsLWbJkCTk5OXTt2pXDhw+Hn3/fvn28\n/vrr7N69m8mTJ2v4RSTBtPhAh1AvPScnh5ycHO6880727t1LTk4OHTp0CF+29vXXX2fmzJkkJSXR\no0cPRo0axdatWzn33HPJyMgIX/b1e9/7HuPHjwcgNTWV7OxsAF599VV27doV3uYXX3wR/qKKH/3o\nR7Rt25a2bdvSvXt39u/fH75kb2VdunShU6dOrF27losvvrjKNxqdPHmS22+/PXwt8/fe++ar/DIz\nM+nbty8Ar732GjNmzAh/5Vznzp3D602dOpVWrVoxYMCAqHxjk4g0LwU634yjv/POOwwaNIjevXvz\nxz/+kXPPPZebb765zsdXvsRuq1atwtOtWrUKj1tXVFTw5ptvkpyc/K2PP32J3tpcd9113Hbbbaxa\ntarK/GXLltGjRw927NhBRUVFle1EelnbynU05zV+RCQ6WvwYOoR66OvXr6dz584kJSXRuXNnjh49\nypYtW8IfiI4YMYJ169Zx6tQpSkpK2Lx5M5mZmRFvY/z48VW+DCI/P79BtU6bNo1f/epXTJgwocr8\n0tJSevbsSatWrXjmmWdq/fq4sWPH8txzz3Ho0CGAKkMuIpLYIuqhm9ke4BhwCih3zqWbWWdgHZAC\n7AGudc4daXRFMbi6WmpqKgcPHuTHP/5xlXllZWXhoYlp06axZcsWhgwZgpnx0EMPcd5557F79+6I\ntvHoo49y2223MXjwYMrLyxk5ciSPP/54vWtt374999xzzxnzb731Vq655hqefvppJk6cWGuvfODA\ngdx3332MGjWKpKQkLrnkkjN6+yKSmCK6fG4Q6OnOuYOV5j0EHHbOPWhmi4FOzrkzk6YSXT5Xmor2\nIw/o8rm1ao7L504BVgf3VwP64kcRkRiKNNAd8IqZ5ZnZ3GBeD+fcvuD+50CPmh5oZnPNLNfMcktK\nShpZroiI1CbSo1yucM7tNbPuwAYzqzJw7JxzZlbj2I1zbiWwEkJDLrWsg5nVo2yRb+iIHJGQiHro\nzrm9we0B4AUgE9hvZj0BgtsDDSkgOTmZQ4cO6Z9SGsQ5x6FDh2o8HFSkpamzh25m5wCtnHPHgvvj\ngd8BLwGzgQeD2xcbUkCvXr0oLi5GwzHSUMnJyTWeiCXS0kQy5NIDeCEYEmkN/D/n3H+b2Vbgb2Y2\nB/gYuLYhBbRp0yZ8FqOIiDRcnYHunPsIGFLD/EPAlU1RlIiI1J/OFBUR8YQCXUTEEwp0ERFPKNBF\nRDyhQBcR8YQCXUTEEwp0ERFPKNBFRDyhQBcR8YQCXUTEEwp0ERFPKNBFRDyhQBcR8YQCXUTEEwp0\nERFPKNBFRDyhQBcR8YQCXUTEEwp0ERFPKNBFRDyhQBcR8YQCXUTEEwp0ERFPKNBFRDyhQBcR8YQC\nXUTEEwp0ERFPRBzoZpZkZtvNbH0w3dfM3jKzD8xsnZmd1XRliohIXerTQ58PFFWaXgosc859HzgC\nzIlmYSIiUj8RBbqZ9QJ+BDwRTBswFsgKVlkNTG2KAkVEJDKR9tCXA78CKoLpLsBR51x5MF0MXBDl\n2kREpB7qDHQzmwQccM7lNWQDZjbXzHLNLLekpKQhTyEiIhGIpId+OTDZzPYAawkNtawAOppZ62Cd\nXsDemh7snFvpnEt3zqV369YtCiWLiEhN6gx059y9zrlezrkU4HrgNefcLCAbmB6sNht4scmqFBGR\nOjXmOPR7gDvN7ANCY+pPRqckERFpiNZ1r/IN59wmYFNw/yMgM/oliYhIQ+hMURERTyjQRUQ8oUAX\nEfGEAl1ExBMKdBERTyjQRUQ8oUAXEfGEAl1ExBMKdBERTyjQRUQ8oUAXEfGEAl1ExBMKdBERTyjQ\nRUQ8oUAXEfGEAl1ExBMKdBERTyjQRUQ8oUAXEfGEAl1ExBMKdBERTyjQRUQ8oUAXEfGEAl1ExBMK\ndBERTyjQRUQ8oUAXEfGEAl1ExBN1BrqZJZvZ22a2w8wKzeyBYH5fM3vLzD4ws3VmdlbTlysiIrWJ\npId+AhjrnBsCpAETzWwYsBRY5pz7PnAEmNN0ZYqISF3qDHQXUhZMtgl+HDAWyArmrwamNkmFIiIS\nkYjG0M0syczygQPABuBD4KhzrjxYpRi4oJbHzjWzXDPLLSkpiUbNIiJSg4gC3Tl3yjmXBvQCMoGL\nIt2Ac26lcy7dOZferVu3BpYpIiJ1qddRLs65o0A2MBzoaGatg0W9gL1Rrk1EROohkqNcuplZx+D+\n2cA4oIhQsE8PVpsNvNhURYqISN1a170KPYHVZpZE6AXgb8659Wa2C1hrZkuA7cCTTViniIjUoc5A\nd87tBC6pYf5HhMbTRUQkDuhMURERTyjQRUQ8oUAXEfGEAl1ExBMKdBERTyjQRUQ8oUAXEfGEAl1E\nxBMKdBERTyjQRUQ8oUAXEfGEAl1ExBMKdBERTyjQRUQ8oUAXEfFEJF9wISINsGzDe1WmF467MEaV\nSEuhHrqIiCcU6CIinlCgi4h4QoEuIuIJBbqIiCcU6CIinlCgi4h4QoEuIuIJBbqIiCcU6CIinlCg\ni4h4os5AN7PeZpZtZrvMrNDM5gfzO5vZBjN7P7jt1PTliohIbSLpoZcDdznnBgDDgNvMbACwGNjo\nnPsBsDGYFhGRGKkz0J1z+5xz24L7x4Ai4AJgCrA6WG01MLWpihQRkbrVawzdzFKAS4C3gB7OuX3B\nos+BHlGtTERE6iXiQDezdsDzwALn3BeVlznnHOBqedxcM8s1s9ySkpJGFSsiIrWLKNDNrA2hMF/j\nnPuPYPZ+M+sZLO8JHKjpsc65lc65dOdcerdu3aJRs4iI1CCSo1wMeBIocs49UmnRS8Ds4P5s4MXo\nlyciIpGK5CvoLgduBN4xs/xg3q+BB4G/mdkc4GPg2qYpUUREIlFnoDvnXgeslsVXRrccERFpKJ0p\nKiLiCQW6iIgnFOgiIp5QoIuIeEKBLiLiiUgOWxRpEZZteK/K9MJxFzbLY0WiRT10ERFPKNBFRDyh\nIRdPaQhApOVRD11ExBMKdBERTyjQRUQ8kThj6Nl/qDo95t7Y1CEiEqfUQxcR8YQCXUTEEwp0ERFP\nKNBFRDyhQBcR8UTiHOUiEid0Fq7EK/XQRUQ8oUAXEfGEAl1ExBMaQ6+s+tmooDNSRSRhqIcuIuIJ\n9dBFvoWOaJFEoh66iIgnFOgiIp7QkIuISITifQiuzh66mT1lZgfMrKDSvM5mtsHM3g9uOzVtmSIi\nUpdIhlxWAROrzVsMbHTO/QDYGEyLiEgM1Tnk4pzbbGYp1WZPAUYH91cDm4B7oliXiFRT/e0+xN9b\nfomtho6h93DO7Qvufw70qG1FM5sLzAX47ne/28DN1cKjr6WL97G5xvK9fSLxoNEfijrnnJm5b1m+\nElgJkJ6eXut6El3DPllZbc7DMalDRJpPQw9b3G9mPQGC2wPRK0lERBqioT30l4DZwIPB7YtRqyiG\ntnx06Ix5w8fEoBCJKg33SEtRZ6Cb2bOEPgDtambFwP8lFOR/M7M5wMfAtU1ZZJPwaPxdxAfqUDVe\nJEe5zKxl0ZVRrkVaGPWcRaJLZ4o2lHr4zUe/a5GIKNAlrtTUa6/+Vlxvw0VqpkBPMBqmkHiWqPtn\notZdXUIHunpuIiLfSOhAjzsa6xU5k/4vmo0CXRKTQkLkDC0i0BN2fKyZQmvLk4uqTA+fo8sENJWE\n3RdboET8W7WIQPdeMwV/Iu7gUge90/GKf4Feww7a2AtVKchCmuOCX7qoWIj2ucQRT38r/wI9AUS6\nAyTqUTzNsYPX9LtJ2OuFN7aXHKtetnr3tYvR78a7QI80BBM1LONNPPVOpHk15n9I+03T8C7QJTr0\nQalUod54QkiYQI+3HnW81SO1q2lcXj1E8VHCBLrv9GFg8/Lp963OhZymQBdvnBnSkAhH4iRCICdC\njaJAb3pNcBhlvPO9fY3RXL+bxgwpaTgqcSnQIxBPARVPtYjEuxpfnJrhA95YvaNRoEdRTX/EuHqr\nWn1HBh2tUIdYvYD69MIdV/8DnlOgi3jIpxeEmjRmWMjnFxgFujRKpMHhe8DUpCW2OVL1+d1EfGa1\nzp1QoEvsKPBEokuBLhIFsblwWeO34/PwQ0ukQPeA/ilFBBToLUr14AeFf0ui48v9p0AXCfg+pq8P\nsP3XKtYFiIhIdKiHLiJSg0R8p9KoQDezicAKIAl4wjn3YFSqEhEJJGKwxkqDA93MkoA/A+OAYmCr\nmb3knNsVreJEROJdPL3gNKaHngl84Jz7CMDM1gJTAAW6SAsVT+HWEjXmQ9ELgE8rTRcH80REJAbM\nOdewB5pNByY6534aTN8I/Jtz7vZq680F5gaT/YF367GZrsDBBhUYf9SW+KS2xB9f2gHRa0sf51y3\nulZqzJDLXqB3pelewbwqnHMrgZrOWa6TmeU659IbVl58UVvik9oSf3xpBzR/Wxoz5LIV+IGZ9TWz\ns4DrgZeiU5aIiNRXg3vozrlyM7sd+B9Chy0+5ZwrjFplIiJSL406Dt0591/Af0Wplpo0aKgmTqkt\n8UltiT++tAOauS0N/lBURETii67lIiLiibgMdDObaGbvmtkHZrY41vXUl5k9ZWYHzKyg0rzOZrbB\nzN4PbjvFssZImFlvM8s2s11mVmhm84P5idiWZDN728x2BG15IJjf18zeCva1dcEH/AnBzJLMbLuZ\nrQ+mE7ItZrbHzN4xs3wzyw3mJdw+BmBmHc0sy8x2m1mRmQ1vzrbEXaBXuqTA/wEGADPNbEBsq6q3\nVcDEavMWAxudcz8ANgbT8a4cuMs5NwAYBtwW/C0SsS0ngLHOuSFAGjDRzIYBS4FlzrnvA0eAOTGs\nsb7mA0WVphO5LWOcc2mVDvFLxH0MQte2+m/n3EXAEEJ/n+Zri3Murn6A4cD/VJq+F7g31nU1oB0p\nQEGl6XeBnsH9nsC7sa6xAW16kdC1exK6LcB3gG3AvxE66aN1ML/KvhfPP4TO+9gIjAXWA5bAbdkD\ndK02L+H2MaAD8L8En03Goi1x10PH30sK9HDO7Qvufw70iGUx9WVmKcAlwFskaFuCIYp84ACwAfgQ\nOOqcKw9WSaR9bTnwK6AimO5C4rbFAa+YWV5wZjkk5j7WFygB/hIMhT1hZufQjG2Jx0D3ngu9VCfM\n4UVm1g54HljgnPui8rJEaotz7pRzLo1Q7zYTuCjGJTWImU0CDjjn8mJdS5Rc4ZwbSmiY9TYzG1l5\nYQLtY62BocC/O+cuAb6k2vBKU7clHgM9oksKJKD9ZtYTILg9EON6ImJmbQiF+Rrn3H8EsxOyLac5\n544C2YSGJTqa2enzMRJlX7scmGxme4C1hIZdVpCYbcE5tze4PQC8QOjFNhH3sWKg2Dn3VjCdRSjg\nm60t8Rjovl5S4CVgdnB/NqHx6LhmZgY8CRQ55x6ptCgR29LNzDoG988m9FlAEaFgnx6slhBtcc7d\n65zr5ZxLIfT/8ZpzbhYJ2BYzO8fM2p++D4wHCkjAfcw59znwqZn1D2ZdSehy4s3Xllh/kFDLhwtX\nAe8RGuO8L9b1NKD+Z4F9wElCr9pzCI1xbgTeB14FOse6zgjacQWht4c7gfzg56oEbctgYHvQlgLg\nt8H8fsDbwAfAc0DbWNdaz3aNBtYnaluCmncEP4Wn/98TcR8L6k4DcoP97D+BTs3ZFp0pKiLiiXgc\nchERkQZQoIuIeEKBLiLiCQW6iIgnFOgiIp5QoIuIeEKBLiLiCQW6iIgn/j9mA2teF7O/ZQAAAABJ\nRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13928e8d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "retweet_vio_mean = exploratory_df_vio.groupby('days_to_event')['retweets'].mean()\n",
    "retweet_non_mean = exploratory_df_non.groupby('days_to_event')['retweets'].mean()\n",
    "retweet_non_mean = handleMissingDays(retweet_non_mean)\n",
    "\n",
    "plt.bar(range(1,61), retweet_vio_mean, alpha=0.5, label='Charlottesville Protest')\n",
    "plt.bar(range(1,61), retweet_non_mean, alpha=0.5, label='Women March')\n",
    "plt.legend(loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Both Charlottesville and Women March tweets were retweeted at a similar rate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGjZJREFUeJzt3Xt0VOW9//H3l4DECnIHUSiBtqJA\nIGLCAZWr5fKzyGUJKkWLSkupWgGvWLv60y66Ki4roO06Lpa1oOUn1Hg8Kj3rHBFDqQaVAAG5eT2o\nIEIAiaBCCXx/f8wwJpCYyeQ28+TzWitrZj97z+zvM9n5zM4ze+8xd0dERFJfo/ouQEREaoYCXUQk\nEAp0EZFAKNBFRAKhQBcRCYQCXUQkEAp0EZFAKNBFRAKhQBcRCUTjulxZ27ZtPSMjoy5XKSKS8tat\nW7fP3dtVtlydBnpGRgYFBQV1uUoRkZRnZh/Fs5yGXEREAqFAFxEJhAJdRCQQdTqGXp5jx46xc+dO\njhw5Ut+lSIpKT0+nU6dONGnSpL5LEalX9R7oO3fupHnz5mRkZGBm9V2OpBh3Z//+/ezcuZOuXbvW\ndzki9areh1yOHDlCmzZtFOaSEDOjTZs2+g9PhCQIdEBhLtWi7UckIikCXUREqq/ex9BPNW/FuzX6\nfLOGn1/pMp999hkzZ85k7dq1tGzZkg4dOjB//nw+/fRTHn74YZYvX57w+u+//36aNWvGnXfeWeEy\nixYtYsSIEZx77rkAzJ8/n2nTpvGd73wn4fWe6je/+Q2DBg3ihz/8IUOGDOHhhx8mOzs7drJX27Zt\n43qeIUOGsHv3btLT02nWrBlPPvkk3bt3j7uOU/taFatWreKMM87gkksuqfJjRRqCpAv0uubujB8/\nnilTprB06VIANm7cyJ49e6r93CUlJXEtt2jRInr16lUm0K+77roaDfTf/va3NfZcS5YsITs7m4UL\nF3LXXXfx4osvlpl//Phx0tLSyn3sqX2tilWrVtGsWTMFeqjyfn9629B7676OFNbgh1zy8vJo0qQJ\n06dPj7X16dOHgQMHAnD48GEmTJjABRdcwOTJk3F3IBKQOTk59OrVi2nTpsXahwwZwsyZM8nOzmbB\nggVl1lVYWEj//v3p3bs348eP5/PPPyc3N5eCggImT55MVlYWCxYs4NNPP2Xo0KEMHToUgJdffpkB\nAwbQt29fJk6cyOHDhwGYPXs2PXr0oHfv3tx5550UFxfTpUsXTpw4AcCXX35J586dOXbsGDfccAO5\nubnf+lr89a9/pV+/fmRlZfHzn/+c48ePf+vygwYN4v333wcil3W455576Nu3L88++2xcff36669Z\nt24dgwcP5uKLL2bkyJHs3r0bgEcffTTWt2uvvZYdO3bw+OOPM2/ePLKysvjnP/9Z+S9XpIFp8IG+\nefNmLr744grnb9iwgfnz57N161Y+/PBDXn/9dQBuvfVW1q5dy+bNm/n666/LDMv861//oqCggDvu\nuKPMc/3kJz9h7ty5bNq0iczMTB544AEmTJhAdnY2S5YsobCwkBkzZnDuueeSl5dHXl4e+/btY86c\nObzyyiusX7+e7OxsHnnkEfbv38/zzz/Pli1b2LRpE7/+9a9p0aIFWVlZ/OMf/wBg+fLljBw5Mq7j\ns7dt28ayZct4/fXXKSwsJC0tjSVLlnzrY1566SUyMzNj023atGH9+vVce+21cfW1cePG/PKXvyQ3\nN5d169Zx0003cd999wHw4IMPsmHDBjZt2sTjjz9ORkYG06dPZ9asWRQWFsbecEXkGw1+yKUy/fr1\no1OnTgBkZWWxY8cOLrvsMvLy8njooYf46quvOHDgAD179uTKK68E4JprrjnteYqLizl48CCDBw8G\nYMqUKUycOLHS9b/xxhts3bqVSy+9FIi8WQwYMIAWLVqQnp7O1KlTGT16NKNHj46te9myZQwdOpSl\nS5dy8803x9XPlStXsm7dOnJycgD4+uuvad++fbnLTp48mTPPPJOMjAwee+yxWPvJfsfb13feeYfN\nmzczfPhwIDJU07FjRwB69+7N5MmTGTduHOPGjYurDyINXYMP9J49e37rUETTpk1j99PS0igpKeHI\nkSPcfPPNFBQU0LlzZ+6///4yx0GfddZZNVafuzN8+HCeeeaZ0+a99dZbrFy5ktzcXP74xz/y6quv\nMmbMGH71q19x4MAB1q1bx7Bhw+Jez5QpU/j978sZxzzFyTH0U1W13+5Oz549WbNmzWnz/v73v7N6\n9Wpeeuklfve73/H2229X6blFGqIGP+QybNgwjh49ysKFC2NtmzZt+tYx2pPh3bZtWw4fPlzp2DRA\nixYtaNWqVex5n3766dgebPPmzTl06FBs2dLT/fv35/XXX4+NVX/55Ze8++67HD58mOLiYq644grm\nzZvHxo0bAWjWrBk5OTnMmDGD0aNHV/jh5Kkuv/xycnNz2bt3LwAHDhzgo4/iumJnwn3t3r07RUVF\nsUA/duwYW7Zs4cSJE3zyyScMHTqUuXPnUlxczOHDh097nUSkrKTbQ4/nMMOaZGY8//zzzJw5k7lz\n55Kenk5GRgbz589n165d5T6mZcuW/OxnP6NXr16cc845sWGKyixevJjp06fz1Vdf0a1bN/7yl78A\ncMMNNzB9+nTOPPNM1qxZw7Rp0xg1alRsLH3RokVMmjSJo0ePAjBnzhyaN2/O2LFjOXLkCO7OI488\nElvPNddcw8SJE1m1alXcr0OPHj2YM2cOI0aM4MSJEzRp0oQ//elPdOnSJe7nSKSvubm53HbbbRQX\nF1NSUsLMmTM5//zzue666yguLsbdue2222jZsiVXXnklEyZM4IUXXuCxxx7TOLrIKezk0Rl1ITs7\n20/9gott27Zx4YUX1lkNEiZtRwHQYYsVMrN17n76OOcpGvyQi4hIKBToIiKBUKCLiARCgS4iEggF\nuohIIBToIiKBSLrj0Ms9dKk6KjnsadasWXTp0oWZM2cCMHLkSDp37swTTzwBwB133MF5553H7bff\nXrN1VVFGRgadO3cuc8JTVlYWJSUlbN68udrPf8MNNzB69GgmTJhQ7ecSkfrR4PfQL730UvLz8wE4\nceIE+/btY8uWLbH5+fn5SXO51kOHDvHJJ58AkeOuqyrey/mKSGpq8IF+ySWXxE4937JlC7169aJ5\n8+Z8/vnnHD16lG3bttG3b1/cnbvuuotevXqRmZnJsmXLgMg1ugcPHszYsWPp1q0bs2fPZsmSJfTr\n14/MzEw++OADAIqKirjqqqvIyckhJycndtXG+++/n5tuuokhQ4bQrVs3Hn300Qprvfrqq2PrfeaZ\nZ5g0aVJs3o4dOxg4cCB9+/alb9++sTepVatWMXDgQMaMGUOPHj0AeOqpp+jduzd9+vTh+uuvjz3H\n6tWrueSSS+jWrVtclzMQkeSSfEMudezcc8+lcePGfPzxx+Tn5zNgwAB27drFmjVraNGiBZmZmZxx\nxhk899xzFBYWsnHjRvbt20dOTg6DBg0CIl+IsW3bNlq3bk23bt346U9/yltvvcWCBQt47LHHmD9/\nPjNmzGDWrFlcdtllfPzxx4wcOTK2l719+3by8vI4dOgQ3bt35xe/+EW5l7y96qqruPHGG7nzzjt5\n6aWXWLJkCU8//TQA7du3Z8WKFaSnp/Pee+8xadIkTp6Vu379ejZv3kzXrl3ZsmULc+bMIT8/n7Zt\n23LgwIHY8+/evZvXXnuN7du3M2bMGA2/iKSYBh/oENlLz8/PJz8/n9tvv51du3aRn59PixYtYpet\nfe2115g0aRJpaWl06NCBwYMHs3btWs4++2xycnJil3393ve+x4gRIwDIzMwkLy8PgFdeeYWtW7fG\n1vnFF1/EvqjiRz/6EU2bNqVp06a0b9+ePXv2xC7ZW1qbNm1o1aoVS5cu5cILLyzzjUbHjh3j1ltv\njV3L/N13v/kqv379+tG1a1cAXn31VSZOnBj7yrnWrVvHlhs3bhyNGjWiR48eNfKNTSJStxTofDOO\n/vbbb9OrVy86d+7MH/7wB84++2xuvPHGSh9f+hK7jRo1ik03atQoNm594sQJ3njjDdLT07/18Scv\n0VuRa665hltuuYVFixaVaZ83bx4dOnRg48aNnDhxosx64r2sbek66vIaPyJSMxr8GDpE9tCXL19O\n69atSUtLo3Xr1hw8eJA1a9bEPhAdOHAgy5Yt4/jx4xQVFbF69Wr69esX9zpGjBhR5ssgCgsLE6p1\n/Pjx3H333YwcObJMe3FxMR07dqRRo0Y8/fTTFX593LBhw3j22WfZv38/QJkhFxFJbXHtoZvZDuAQ\ncBwocfdsM2sNLAMygB3A1e7+ebUrqoerq2VmZrJv3z5+/OMfl2k7fPhwbGhi/PjxrFmzhj59+mBm\nPPTQQ5xzzjls3749rnU8+uij3HLLLfTu3ZuSkhIGDRrE448/XuVamzdvzj333HNa+80338xVV13F\nU089xahRoyrcK+/Zsyf33XcfgwcPJi0tjYsuuui0vX0RSU1xXT43GujZ7r6vVNtDwAF3f9DMZgOt\n3P30pClFl8+V2qLtKAC6fG6F6uLyuWOBxdH7iwF98aOISD2KN9AdeNnM1pnZtGhbB3ffHb3/GdCh\nvAea2TQzKzCzgqKiomqWKyIiFYn3KJfL3H2XmbUHVphZmYFjd3czK3fsxt0XAgshMuRSwTKYWRXK\nFvmGjsgRiYhrD93dd0Vv9wLPA/2APWbWESB6uzeRAtLT09m/f7/+KCUh7s7+/fvLPRxUpKGpdA/d\nzM4CGrn7oej9EcBvgReBKcCD0dsXEimgU6dO7Ny5Ew3HSKLS09PLPRFLpKGJZ8ilA/B8dEikMfD/\n3P2/zWwt8Dczmwp8BFydSAFNmjSJncUoIiKJqzTQ3f1DoE857fuBy2ujKBERqTqdKSoiEggFuohI\nIBToIiKBUKCLiARCgS4iEggFuohIIBToIiKBUKCLiARCgS4iEggFuohIIBToIiKBUKCLiARCgS4i\nEggFuohIIBToIiKBUKCLiARCgS4iEggFuohIIBToIiKBUKCLiARCgS4iEggFuohIIBToIiKBUKCL\niARCgS4iEggFuohIIBToIiKBiDvQzSzNzDaY2fLodFcze9PM3jezZWZ2Ru2VKSIilanKHvoMYFup\n6bnAPHf/PvA5MLUmCxMRkaqJK9DNrBPwI+CJ6LQBw4Dc6CKLgXG1UaCIiMQn3j30+cDdwInodBvg\noLuXRKd3AufVcG0iIlIFlQa6mY0G9rr7ukRWYGbTzKzAzAqKiooSeQoREYlDPHvolwJjzGwHsJTI\nUMsCoKWZNY4u0wnYVd6D3X2hu2e7e3a7du1qoGQRESlPpYHu7ve6eyd3zwCuBV5198lAHjAhutgU\n4IVaq1JERCpVnePQ7wFuN7P3iYyp/7lmShIRkUQ0rnyRb7j7KmBV9P6HQL+aL0lERBKhM0VFRAKh\nQBcRCYQCXUQkEAp0EZFAKNBFRAKhQBcRCYQCXUQkEAp0EZFAKNBFRAKhQBcRCYQCXUQkEAp0EZFA\nKNBFRAKhQBcRCYQCXUQkEAp0EZFAKNBFRAKhQBcRCYQCXUQkEAp0EZFAKNBFRAKhQBcRCYQCXUQk\nEAp0EZFAKNBFRAKhQBcRCYQCXUQkEAp0EZFAVBroZpZuZm+Z2UYz22JmD0Tbu5rZm2b2vpktM7Mz\nar9cERGpSDx76EeBYe7eB8gCRplZf2AuMM/dvw98DkytvTJFRKQylQa6RxyOTjaJ/jgwDMiNti8G\nxtVKhSIiEpe4xtDNLM3MCoG9wArgA+Cgu5dEF9kJnFfBY6eZWYGZFRQVFdVEzSIiUo64At3dj7t7\nFtAJ6AdcEO8K3H2hu2e7e3a7du0SLFNERCpTpaNc3P0gkAcMAFqaWePorE7ArhquTUREqiCeo1za\nmVnL6P0zgeHANiLBPiG62BTghdoqUkREKte48kXoCCw2szQibwB/c/flZrYVWGpmc4ANwJ9rsU4R\nEalEpYHu7puAi8pp/5DIeLqIiCQBnSkqIhIIBbqISCAU6CIigVCgi4gEQoEuIhIIBbqISCAU6CIi\ngVCgi4gEQoEuIhIIBbqISCAU6CIigVCgi4gEQoEuIhIIBbqISCAU6CIigVCgi4gEQoEuIhIIBbqI\nSCAU6CIigVCgi4gEQoEuIhIIBbqISCAU6CIigVCgi4gEQoEuIhIIBbqISCAU6CIigag00M2ss5nl\nmdlWM9tiZjOi7a3NbIWZvRe9bVX75YqISEXi2UMvAe5w9x5Af+AWM+sBzAZWuvsPgJXRaRERqSeV\nBrq773b39dH7h4BtwHnAWGBxdLHFwLjaKlJERCpXpTF0M8sALgLeBDq4++7orM+ADjVamYiIVEnc\ngW5mzYDngJnu/kXpee7ugFfwuGlmVmBmBUVFRdUqVkREKhZXoJtZEyJhvsTd/yPavMfMOkbndwT2\nlvdYd1/o7tnunt2uXbuaqFlERMoRz1EuBvwZ2Obuj5Sa9SIwJXp/CvBCzZcnIiLxahzHMpcC1wNv\nm1lhtO1XwIPA38xsKvARcHXtlCgiIvGoNNDd/TXAKph9ec2WIyIiidKZoiIigVCgi4gEQoEuIhII\nBbqISCAU6CIigVCgi4gEQoEuIhIIBbqISCAU6CIigVCgi4gEQoEuIhIIBbqISCDiudpicsj7fdnp\noffWTx0iIklKe+giIoFQoIuIBEKBLiISCAW6iEggFOgiIoFQoIuIBEKBLiISCAW6iEggFOgiIoFI\nnTNF68KpZ6OCzkgVkZShPXQRkUAo0EVEAqFAFxEJhAJdRCQQCnQRkUBUGuhm9qSZ7TWzzaXaWpvZ\nCjN7L3rbqnbLFBGRysSzh74IGHVK22xgpbv/AFgZnRYRkXpUaaC7+2rgwCnNY4HF0fuLgXE1XJeI\niFRRoicWdXD33dH7nwEdKlrQzKYB0wC++93vJri6Cuhr6UREYqp9pqi7u5n5t8xfCCwEyM7OrnA5\nqWF6sxNpcBI9ymWPmXUEiN7urbmSREQkEYkG+ovAlOj9KcALNVOOiIgkqtIhFzN7BhgCtDWzncD/\nBR4E/mZmU4GPgKtrs8haoSEJEQlMpYHu7pMqmHV5DdciIiLVoMvnJkp7+HVHr7VIXHTqv4hIIBTo\nIiKBUKCLiARCY+g1SWO9IqfT30WdUaBLalJIiJxGgZ7M6iq0FI4iQVCgh0CBLInSthOU8AK9vA1U\nG23NqIvXUb8rkYSFF+giqaa6b2L19SaoN9+K1dNro8MWRUQCoT10KZ/2vqQ0bQ8pQYEu4VMYSQOh\nQE8WCp26pddbAqRAl3CcGtKgI3GkQVGg17aGeBhl6P2rDr02UosU6PFIpj/CZKpFJBUF/DekQG9I\n6mpIIiQ6xltSiAJdJER6Q2iQFOhSPfEGR0MMmIbY53jVxmuj11uBLvVIf4AiNUqBLlIT6uPCZbW1\nHklZupaLiEggFOgiIoHQkIvISaGP6esD7OBpD11EJBDaQxcRKU8K/qdSrUA3s1HAAiANeMLdH6yR\nqkRETkrBYK0vCQe6maUBfwKGAzuBtWb2ortvraniRESSXhK94VRnD70f8L67fwhgZkuBsYACXaQO\nrflwf5npAUPrqRBIqnCrinkr3i0zPWv4+cn1usapOoF+HvBJqemdwL9Vr5yKlffi1nZbXa2nvtqS\nrR71L7H+lSdZaqyt31+8AVzTQR3v89XXm4G5e2IPNJsAjHL3n0anrwf+zd1vPWW5acC06GR34J0q\nrKYtsC+hApOP+pKc1JfkE0o/oOb60sXd21W2UHX20HcBnUtNd4q2leHuC4GFiazAzArcPTux8pKL\n+pKc1JfkE0o/oO77Up3j0NcCPzCzrmZ2BnAt8GLNlCUiIlWV8B66u5eY2a3A/xA5bPFJd99SY5WJ\niEiVVOs4dHf/L+C/aqiW8iQ0VJOk1JfkpL4kn1D6AXXcl4Q/FBURkeSia7mIiAQiKQPdzEaZ2Ttm\n9r6Zza7veqrKzJ40s71mtrlUW2szW2Fm70VvW9VnjfEws85mlmdmW81si5nNiLanYl/SzewtM9sY\n7csD0fauZvZmdFtbFv2APyWYWZqZbTCz5dHplOyLme0ws7fNrNDMCqJtKbeNAZhZSzPLNbPtZrbN\nzAbUZV+SLtBLXVLg/wA9gElm1qN+q6qyRcCoU9pmAyvd/QfAyuh0sisB7nD3HkB/4Jbo7yIV+3IU\nGObufYAsYJSZ9QfmAvPc/fvA58DUeqyxqmYA20pNp3Jfhrp7VqlD/FJxG4PIta3+290vAPoQ+f3U\nXV/cPal+gAHA/5Savhe4t77rSqAfGcDmUtPvAB2j9zsC79R3jQn06QUi1+5J6b4A3wHWEzmzeR/Q\nONpeZttL5h8i532sBIYBywFL4b7sANqe0pZy2xjQAvhfop9N1kdfkm4PnfIvKXBePdVSkzq4++7o\n/c+ADvVZTFWZWQZwEfAmKdqX6BBFIbAXWAF8ABx095LoIqm0rc0H7gZORKfbkLp9ceBlM1sXPbMc\nUnMb6woUAX+JDoU9YWZnUYd9ScZAD55H3qpT5vAiM2sGPAfMdPcvSs9Lpb64+3F3zyKyd9sPuKCe\nS0qImY0G9rr7uvqupYZc5u59iQyz3mJmg0rPTKFtrDHQF/h3d78I+JJThldquy/JGOhxXVIgBe0x\ns44A0du99VxPXMysCZEwX+Lu/xFtTsm+nOTuB4E8IsMSLc3s5PkYqbKtXQqMMbMdwFIiwy4LSM2+\n4O67ord7geeJvNmm4ja2E9jp7m9Gp3OJBHyd9SUZAz3USwq8CEyJ3p9CZDw6qZmZAX8Gtrn7I6Vm\npWJf2plZy+j9M4l8FrCNSLBPiC6WEn1x93vdvZO7ZxD5+3jV3SeTgn0xs7PMrPnJ+8AIYDMpuI25\n+2fAJ2bWPdp0OZHLidddX+r7g4QKPly4AniXyBjnffVdTwL1PwPsBo4RedeeSmSMcyXwHvAK0Lq+\n64yjH5cR+fdwE1AY/bkiRfvSG9gQ7ctm4DfR9m7AW8D7wLNA0/qutYr9GgIsT9W+RGveGP3ZcvLv\nPRW3sWjdWUBBdDv7T6BVXfZFZ4qKiAQiGYdcREQkAQp0EZFAKNBFRAKhQBcRCYQCXUQkEAp0EZFA\nKNBFRAKhQBcRCcT/BwTsavXaKF8QAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13a1420b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mention_vio_mean = exploratory_df_vio.groupby('days_to_event')['mention_count'].mean()\n",
    "mention_non_mean = exploratory_df_non.groupby('days_to_event')['mention_count'].mean()\n",
    "mention_non_mean = handleMissingDays(retweet_non_mean)\n",
    "\n",
    "plt.bar(range(1,61), mention_vio_mean, alpha=0.5, label='Charlottesville Protest')\n",
    "plt.bar(range(1,61), mention_non_mean, alpha=0.5, label='Women March')\n",
    "plt.legend(loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There were hardly any mentions for Charlotteville protest tweets. However, on average mention count for Women March event tweets was on an upward trend leading upto the event."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGbBJREFUeJzt3X10VPW97/H3l4DECoI8iChIoOeI\nAoGICUdUHpWHYxFlCVVKLahtStUK+Gzt6rJddFlcVoLWdVwsj1JtrnKNx9sj7e1RMZRqUEkwYCD4\neKmCKAE0EhUk8r1/ZBgJJGQnZGbySz6vtbIye8+e2d/fZM9nfvnthzF3R0REwtEu1QWIiEjjKLhF\nRAKj4BYRCYyCW0QkMApuEZHAKLhFRAKj4BYRCYyCW0QkMApuEZHAtE/Ek/bo0cMzMjIS8dQiIq1S\nSUnJTnfvGWXZhAR3RkYGxcXFiXhqEZFWycz+GXVZDZWIiARGwS0iEhgFt4hIYBIyxl2X/fv3s3Xr\nVvbu3ZusVUork56eTp8+fejQoUOqSxFJqaQF99atW+ncuTMZGRmYWbJWK62Eu7Nr1y62bt1K//79\nU12OSEpFGioxs65mVmBmm82s3MxGNnZFe/fupXv37gptaRIzo3v37vqPTYToPe4lwN/cfbqZHQd8\npykrU2jLsdD2I1KjweA2sy7AaGAOgLt/DXyd2LJERKQ+UXrc/YEK4DEzGwaUAPPc/YtjWfHiF94+\nlocfYcGEMxpc5uOPP2b+/PmsXbuWrl270qtXL/Ly8vjoo4+47777WLFiRZPXf/fdd9OpUyduueWW\nepdZtmwZEydO5NRTTwUgLy+P3NxcvvOdJv0DU6df/epXjB49mosuuoixY8dy3333kZ2dHT8pqkeP\nHpGeZ+zYsWzfvp309HQ6derEo48+ysCBAyPXcXhbG2PVqlUcd9xxnHfeeY1+rEhbEGWMuz0wHPgP\ndz8b+AK44/CFzCzXzIrNrLiioqKZyzx27s60adMYO3Ys7733HiUlJdxzzz188sknx/zc1dXVkZZb\ntmwZH330UXw6Ly+PL7/88pjXf6jf/OY3XHTRRc3yXPn5+axfv57Zs2dz6623HnH/N998U+9jD29r\nY6xatYqioqImPVbaqMJ7av+0clGCeyuw1d1fi00XUBPktbj7UnfPdvfsnj0jnW6fVIWFhXTo0IG5\nc+fG5w0bNoxRo0YBUFVVxfTp0znzzDOZNWsW7g7UBGFOTg5DhgwhNzc3Pn/s2LHMnz+f7OxslixZ\nUmtdpaWlnHvuuQwdOpRp06bx6aefUlBQQHFxMbNmzSIrK4slS5bw0UcfMW7cOMaNGwfA888/z8iR\nIxk+fDgzZsygqqoKgDvuuINBgwYxdOhQbrnlFiorK+nXrx8HDhwA4IsvvqBv377s37+fOXPmUFBQ\ncNTX4k9/+hMjRowgKyuLn/70p0cNYIDRo0fz7rvvAjWXM7j99tsZPnw4Tz/9dKS2fvXVV5SUlDBm\nzBjOOeccJk2axPbt2wF44IEH4m278sor2bJlCw8//DCLFy8mKyuLf/zjHw3/cUXamAaD290/Bj40\ns4P/J18IbEpoVQlQVlbGOeecU+/9b7zxBnl5eWzatIn333+fV155BYAbbriBtWvXUlZWxldffVVr\nOOXrr7+muLiYm2++udZz/ehHP2LRokVs2LCBzMxMfv3rXzN9+nSys7PJz8+ntLSUefPmceqpp1JY\nWEhhYSE7d+5k4cKFvPjii6xbt47s7Gzuv/9+du3axbPPPsvGjRvZsGEDv/zlL+nSpQtZWVn8/e9/\nB2DFihVMmjQp0vHN5eXlLF++nFdeeYXS0lLS0tLIz88/6mOee+45MjMz49Pdu3dn3bp1XHnllZHa\n2r59e37+859TUFBASUkJ11xzDXfddRcAv/vd73jjjTfYsGEDDz/8MBkZGcydO5cFCxZQWloa/2AV\nkW9FPark50B+7IiS94GrE1dSaowYMYI+ffoAkJWVxZYtW7jgggsoLCzk3nvv5csvv2T37t0MHjyY\nSy65BIArrrjiiOeprKzks88+Y8yYMQDMnj2bGTNmNLj+V199lU2bNnH++ecDNR8KI0eOpEuXLqSn\np3PttdcyZcoUpkyZEl/38uXLGTduHE899RTXXXddpHauXLmSkpIScnJyAPjqq684+eST61x21qxZ\nHH/88WRkZPDggw/G5x9sd9S2vvXWW5SVlTFhwgSgZoild+/eAAwdOpRZs2Zx2WWXcdlll0Vqg0hb\nFym43b0UyE5wLQk1ePDgow4hdOzYMX47LS2N6upq9u7dy3XXXUdxcTF9+/bl7rvvrnUc8QknnNBs\n9bk7EyZM4Mknnzzivtdff52VK1dSUFDAH/7wB1566SWmTp3KL37xC3bv3k1JSQnjx4+PvJ7Zs2dz\nzz0NjwPm5+eTnX3kn72x7XZ3Bg8ezJo1a4647y9/+QurV6/mueee47e//S1vvvlmo55bpC1qM9cq\nGT9+PPv27WPp0qXxeRs2bDjqGOrBkO7RowdVVVUNjh0DdOnShZNOOin+vE888US8R9q5c2f27NkT\nX/bQ6XPPPZdXXnklPpb8xRdf8Pbbb1NVVUVlZSUXX3wxixcvZv369QB06tSJnJwc5s2bx5QpU0hL\nS4v0Olx44YUUFBSwY8cOAHbv3s0//xn5apJNauvAgQOpqKiIB/f+/fvZuHEjBw4c4MMPP2TcuHEs\nWrSIyspKqqqqjnidRKS2pJ3yfrgoh+81JzPj2WefZf78+SxatIj09HQyMjLIy8tj27ZtdT6ma9eu\n/OQnP2HIkCGccsop8eGFhvzxj39k7ty5fPnllwwYMIDHHnsMgDlz5jB37lyOP/541qxZQ25uLpMn\nT46PdS9btoyZM2eyb98+ABYuXEjnzp259NJL2bt3L+7O/fffH1/PFVdcwYwZM1i1alXk12HQoEEs\nXLiQiRMncuDAATp06MBDDz1Ev379Ij9HU9paUFDAjTfeSGVlJdXV1cyfP58zzjiDH/7wh1RWVuLu\n3HjjjXTt2pVLLrmE6dOn8+c//5kHH3xQ49wih7GDR0k0p+zsbD/8ixTKy8s566yzmn1d0rZoO5I6\nHX4I4Lg7U1PHMTCzEnePNCTdZoZKRERaCwW3iEhgFNwiIoFRcIuIBEbBLSISGAW3iEhgUnYcd7Nf\nwauBw38WLFhAv379mD9/PgCTJk2ib9++PPLIIwDcfPPNnHbaadx0003NW1cjZWRk0Ldv31onBmVl\nZVFdXU1ZWdkxP/+cOXOYMmUK06dPP+bnEpHUaDM97vPPPz9+qdADBw6wc+dONm7cGL+/qKioxVz/\nec+ePXz44YdAzXHLjRX1MrMiEqY2E9znnXde/JTrjRs3MmTIEDp37synn37Kvn37KC8vZ/jw4bg7\nt956K0OGDCEzM5Ply5cDNdeIHjNmDJdeeikDBgzgjjvuID8/nxEjRpCZmcl7770HQEVFBZdffjk5\nOTnk5OTErzJ49913c8011zB27FgGDBjAAw88UG+t3//+9+PrffLJJ5k5c2b8vi1btjBq1CiGDx/O\n8OHD4x9Gq1atYtSoUUydOpVBgwYB8PjjjzN06FCGDRvGVVddFX+O1atXc9555zFgwIBIp/GLSMuS\nuqGSJDv11FNp3749H3zwAUVFRYwcOZJt27axZs0aunTpQmZmJscddxzPPPMMpaWlrF+/np07d5KT\nk8Po0aMBWL9+PeXl5XTr1o0BAwbw4x//mNdff50lS5bw4IMPkpeXx7x581iwYAEXXHABH3zwAZMm\nTYr3mjdv3kxhYSF79uxh4MCB/OxnP6vzUqyXX345V199NbfccgvPPfcc+fn5PPHEEwCcfPLJvPDC\nC6Snp/POO+8wc+ZMDp6lum7dOsrKyujfvz8bN25k4cKFFBUV0aNHD3bv3h1//u3bt/Pyyy+zefNm\npk6dqmETkcC0meCGml53UVERRUVF3HTTTWzbto2ioiK6dOkSv5zqyy+/zMyZM0lLS6NXr16MGTOG\ntWvXcuKJJ5KTkxO/HOl3v/tdJk6cCEBmZiaFhYUAvPjii2za9O3lyj///PP4FyJ873vfo2PHjnTs\n2JGTTz6ZTz75JH4p2UN1796dk046iaeeeoqzzjqr1leb7d+/nxtuuCF+Le233/72K+BGjBhB//79\nAXjppZeYMWNG/KvKunXrFl/usssuo127dgwaNKhZvgFIRJKrTQX3wXHuN998kyFDhtC3b19+//vf\nc+KJJ3L11Q1fYvzQS7+2a9cuPt2uXbv4uPKBAwd49dVXSU9PP+rjD146tj5XXHEF119/PcuWLas1\nf/HixfTq1Yv169dz4MCBWuuJernVQ+tIxLVqRCSx2swYN9T0uFesWEG3bt1IS0ujW7dufPbZZ6xZ\nsya+Y3LUqFEsX76cb775hoqKClavXs2IESMir2PixIm1vnSgtLS0SbVOmzaN2267jUmTJtWaX1lZ\nSe/evWnXrh1PPPFEvV87Nn78eJ5++ml27doFUGuoRETClroedwqu3pWZmcnOnTv5wQ9+UGteVVVV\nfEhh2rRprFmzhmHDhmFm3HvvvZxyyils3rw50joeeOABrr/+eoYOHUp1dTWjR4/m4YcfbnStnTt3\n5vbbbz9i/nXXXcfll1/O448/zuTJk+vtZQ8ePJi77rqLMWPGkJaWxtlnn31E711EwqTLukpQtB1J\nnXRZVxERackU3CIigUlqcOsIBjkW2n5EaiQtuNPT09m1a5fefNIk7s6uXbvqPMxSpK1J2lElffr0\nYevWrVRUVCRrldLKpKen13nCkkhbk7Tg7tChQ/ysPhERaTrtnBQRCYyCW0QkMJGGSsxsC7AH+Aao\njnqQuIiINL/GjHGPc/edCatEREQi0VCJiEhgoga3A8+bWYmZ5da1gJnlmlmxmRXrkD8RkcSJGtwX\nuPtw4N+B681s9OELuPtSd8929+yePXs2a5EiIvKtSMHt7ttiv3cAzwLRL1AtIiLNqsHgNrMTzKzz\nwdvARKAs0YWJiEjdohxV0gt41swOLv+/3P1vCa1KRETq1WBwu/v7wLAk1CIiIhHocEARkcAouEVE\nAqPgFhEJjIJbRCQwCm4RkcAouEVEApO0b8AREUmqwntqT4+7MzV1JIB63CIigVFwi4gERsEtIhIY\nBbeISGAU3CIigVFwi4gERsEtIhIYBbeISGAU3CIigVFwi4gERsEtIhIYBbeISGAU3CIigVFwi4gE\nRsEtIhIYBbeISGAU3CIigVFwi4gEJnJwm1mamb1hZisSWZCIiBxdY3rc84DyRBUiIiLRRApuM+sD\nfA94JLHliIhIQ6L2uPOA24ADCaxFREQiaDC4zWwKsMPdSxpYLtfMis2suKKiotkKFBGR2qL0uM8H\npprZFuApYLyZ/enwhdx9qbtnu3t2z549m7lMERE5qMHgdvc73b2Pu2cAVwIvufsPE16ZiIjUScdx\ni4gEpn1jFnb3VcCqhFQiIiKRqMctIhIYBbeISGAU3CIigVFwi4gERsEtIhIYBbeISGAU3CIigVFw\ni4gERsEtIhIYBbeISGAU3CIigVFwi4gERsEtIhIYBbeISGAU3CIigVFwi4gERsEtIhIYBbeISGAU\n3CIigVFwi4gERsEtIhIYBbeISGAU3CIigVFwi4gERsEtIhIYBbeISGAaDG4zSzez181svZltNLNf\nJ6MwERGpW/sIy+wDxrt7lZl1AF42s//r7q8muDYREalDg8Ht7g5UxSY7xH48kUWJiEj9ovS4MbM0\noAT4F+Ahd3+tjmVygVyA008/vTlrFBGpUXjPkfPG3Zn8Og46vJ4k1RJp56S7f+PuWUAfYISZDalj\nmaXunu3u2T179mzuOkVEJKZRR5W4+2dAITA5MeWIiEhDohxV0tPMusZuHw9MADYnujAREalblDHu\n3sAfY+Pc7YD/7e4rEluWiIjUJ8pRJRuAs5NQi0g0KdohJNJS6MxJEZHARDocUEQC0dIOl5OEUI9b\nRCQwCm4RkcBoqEQklbSjVZpAPW4RkcCoxy3RqGcobV0Leg8ouKU2HZXwrRb0RhU5lIJbRBpPH2op\npeCW1kFBIm2IglukrdKHXbDaZnBrg5VE0zYmCdQ2g1uSSyEm0qwU3C1BcwebglKkVVNwS/Nq6R8a\n9R3uWEfda97fVWvWyHEJrEuaXWv++ym4W6qWHoBtlf4u9Vr8wtu1phdMOCNFlbR+LS+4j+WNkco3\nld7QIkdQmCdGywtuabvq+vBrYR+Irfnfb/lWS//AUXC3FS0sAJNST7La3NJeW6lXSw/kqBTcoVNo\niLQ5Cu6jaWmhmMJ66hwiaGmvj0gThNgLDze4o4aGdljWK8QNNmgtfHtIFm13xy6M4NYGnxDnfrD0\nsDn3paSO0DX7DssAtvdk7KRtTMC3tZ3GYQR3MmhHlrRkdWw36rm2XQruplD4SisQYi/18JohjLqb\nW4PBbWZ9gceBXoADS919SaILE4EwwyVZEjLUpU5JEKL0uKuBm919nZl1BkrM7AV335Tg2kQkJuoH\nmHqkbUODwe3u24Htsdt7zKwcOA1QcAdEPVdpqqhj6drZnTyNGuM2swzgbOC1RBSTCK0psFpTW0Sk\n6SIHt5l1Ap4B5rv753XcnwvkApx++unNVqCININjGLtORk9aR8g0TqTgNrMO1IR2vrv/V13LuPtS\nYClAdna2N1uFklQhjJE2dby3pbVDpKmiHFViwH8C5e5+f+JLktZGQVu/utqciNehLb62rVmUHvf5\nwFXAm2ZWGpv3C3f/a+LKkhC0pjBoTW05FnodwhDlqJKXAUtCLUHTBt84er1Emq5dqgsQEZHG0Snv\nrZB6syKtm4JbJEn0gSrNpVUFdwhvjOauMYQ2i0jzalXBLdKQEI5TF2lImwzuEHqpIdQo9dPfTxKp\nTQZ3XXTSQ9j0WktbouCWZqUAFUk8HcctIhIY9bhbAPVSRVq+ZF1XJopgg1thJ9Ly6csVEiOI4FZI\ni4h8K4jgThV9YIg0v7p64eqZN452ToqIBKbF9bjVyxUROTr1uEVEAtPietwh0H8FIpJK6nGLiARG\nwS0iEhgFt4hIYBTcIiKBUXCLiARGwS0iEhgFt4hIYBTcIiKBUXCLiASmweA2s0fNbIeZlSWjIBER\nObooPe5lwOQE1yEiIhE1GNzuvhrYnYRaREQkgma7yJSZ5QK5AKeffnpzPa2ISEKF+CUOzbZz0t2X\nunu2u2f37NmzuZ5WREQOo6NKREQCo+AWEQlMlMMBnwTWAAPNbKuZXZv4skREpD4N7px095nJKERE\nRKLRUImISGAU3CIigVFwi4gERsEtIhIYBbeISGAU3CIigVFwi4gERsEtIhIYBbeISGAU3CIigVFw\ni4gERsEtIhIYBbeISGAU3CIigVFwi4gERsEtIhIYBbeISGAU3CIigVFwi4gERsEtIhIYBbeISGAU\n3CIigVFwi4gERsEtIhIYBbeISGAiBbeZTTazt8zsXTO7I9FFiYhI/RoMbjNLAx4C/h0YBMw0s0GJ\nLkxEROoWpcc9AnjX3d9396+Bp4BLE1uWiIjUJ0pwnwZ8eMj01tg8ERFJAXP3oy9gNh2Y7O4/jk1f\nBfybu99w2HK5QG5sciDwViPq6AHsbMTyLZna0vK0lnaA2tJSNUdb+rl7zygLto+wzDag7yHTfWLz\nanH3pcDSSOUdxsyK3T27KY9tadSWlqe1tAPUlpYq2W2JMlSyFvhXM+tvZscBVwL/ndiyRESkPg32\nuN292sxuAP4HSAMedfeNCa9MRETqFGWoBHf/K/DXBNbRpCGWFkptaXlaSztAbWmpktqWBndOiohI\ny6JT3kVEApPS4A75VHoze9TMdphZ2SHzupnZC2b2Tuz3SamsMSoz62tmhWa2ycw2mtm82Pzg2mNm\n6Wb2upmtj7Xl17H5/c3stdi2tjy2oz0IZpZmZm+Y2YrYdJBtMbMtZvammZWaWXFsXnDbGICZdTWz\nAjPbbGblZjYymW1JWXC3glPplwGTD5t3B7DS3f8VWBmbDkE1cLO7DwLOBa6P/S1CbM8+YLy7DwOy\ngMlmdi6wCFjs7v8CfApcm8IaG2seUH7IdMhtGefuWYccOhfiNgawBPibu58JDKPm75O8trh7Sn6A\nkcD/HDJ9J3BnquppYhsygLJDpt8Cesdu9wbeSnWNTWzXn4EJobcH+A6wDvg3ak6OaB+bX2vba8k/\n1Jw3sRIYD6wALOC2bAF6HDYvuG0M6AL8P2L7CFPRllQOlbTGU+l7ufv22O2PgV6pLKYpzCwDOBt4\njUDbExtaKAV2AC8A7wGfuXt1bJGQtrU84DbgQGy6O+G2xYHnzawkdqY1hLmN9QcqgMdiQ1iPmNkJ\nJLEt2jmZIF7zsRvUITtm1gl4Bpjv7p8fel9I7XH3b9w9i5re6gjgzBSX1CRmNgXY4e4lqa6lmVzg\n7sOpGR693sxGH3pnQNtYe2A48B/ufjbwBYcNiyS6LakM7kin0gfmEzPrDRD7vSPF9URmZh2oCe18\nd/+v2Oxg2wPg7p8BhdQMJ3Q1s4PnLYSyrZ0PTDWzLdRclXM8NWOrIbYFd98W+70DeJaaD9UQt7Gt\nwFZ3fy02XUBNkCetLakM7tZ4Kv1/A7Njt2dTM1bc4pmZAf8JlLv7/YfcFVx7zKynmXWN3T6emrH6\ncmoCfHpssSDa4u53unsfd8+g5v3xkrvPIsC2mNkJZtb54G1gIlBGgNuYu38MfGhmA2OzLgQ2kcy2\npHiQ/2LgbWrGIO9K9U6HRtb+JLAd2E/NJ/C11Iw/rgTeAV4EuqW6zohtuYCaf+s2AKWxn4tDbA8w\nFHgj1pYy4Fex+QOA14F3gaeBjqmutZHtGgusCLUtsZrXx342Hny/h7iNxerOAopj29n/AU5KZlt0\n5qSISGC0c1JEJDAKbhGRwCi4RUQCo+AWEQmMgltEJDAKbhGRwCi4RUQCo+AWEQnM/wc2KQG9iO9R\nZgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x139fabe80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "hashtag_vio_mean = exploratory_df_vio.groupby('days_to_event')['hashtag_count'].mean()\n",
    "hashtag_non_mean = exploratory_df_non.groupby('days_to_event')['hashtag_count'].mean()\n",
    "hashtag_non_mean = handleMissingDays(hashtag_non_mean)\n",
    "\n",
    "plt.bar(range(1,61), hashtag_vio_mean, alpha=0.5, label='Charlottesville Protest')\n",
    "plt.bar(range(1,61), hashtag_non_mean, alpha=0.5, label='Women March')\n",
    "plt.legend(loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hashtag count does not seem very interesting for analysis."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Another interesting thing is that Women March tweets got a lot of attention (mentions, retweets, favorites) on the 5oth day before the event for some reason."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split Train data and Test data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>days_to_event</th>\n",
       "      <th>favorites</th>\n",
       "      <th>retweets</th>\n",
       "      <th>mention_count</th>\n",
       "      <th>hashtag_count</th>\n",
       "      <th>pos_emoji</th>\n",
       "      <th>neg_emoji</th>\n",
       "      <th>punc_excl</th>\n",
       "      <th>punc_ques</th>\n",
       "      <th>vaderTextScore</th>\n",
       "      <th>...</th>\n",
       "      <th>1845</th>\n",
       "      <th>1846</th>\n",
       "      <th>1847</th>\n",
       "      <th>1848</th>\n",
       "      <th>1849</th>\n",
       "      <th>1850</th>\n",
       "      <th>1851</th>\n",
       "      <th>1852</th>\n",
       "      <th>1853</th>\n",
       "      <th>1854</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>43492</th>\n",
       "      <td>18</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.6705</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48760</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15962</th>\n",
       "      <td>59</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21152</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13258</th>\n",
       "      <td>56</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.9169</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1867 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       days_to_event  favorites  retweets  mention_count  hashtag_count  \\\n",
       "43492             18          7         6              0              2   \n",
       "48760              1          2         0              0              2   \n",
       "15962             59          1         0              1             12   \n",
       "21152              7          0         0              0              1   \n",
       "13258             56          0         0              1              1   \n",
       "\n",
       "       pos_emoji  neg_emoji  punc_excl  punc_ques  vaderTextScore  ...   1845  \\\n",
       "43492          0          0          0          0          0.6705  ...    0.0   \n",
       "48760          0          0          0          0          0.0000  ...    0.0   \n",
       "15962          0          0          1          0          0.0000  ...    0.0   \n",
       "21152          0          0          0          0          0.0000  ...    0.0   \n",
       "13258          0          0          0          0         -0.9169  ...    0.0   \n",
       "\n",
       "       1846  1847  1848  1849  1850  1851  1852  1853  1854  \n",
       "43492   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "48760   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "15962   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "21152   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "13258   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "\n",
       "[5 rows x 1867 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# shuffle dataset\n",
    "final_training_features = final_training_features_df.drop(['pos_emoji', 'neg_emoji'], axis=1)\n",
    "final_training_features_df = final_training_features_df.sample(frac=1, random_state=np.random.seed(1))\n",
    "final_training_features_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>days_to_event</th>\n",
       "      <th>favorites</th>\n",
       "      <th>retweets</th>\n",
       "      <th>mention_count</th>\n",
       "      <th>hashtag_count</th>\n",
       "      <th>pos_emoji</th>\n",
       "      <th>neg_emoji</th>\n",
       "      <th>punc_excl</th>\n",
       "      <th>punc_ques</th>\n",
       "      <th>vaderTextScore</th>\n",
       "      <th>...</th>\n",
       "      <th>1845</th>\n",
       "      <th>1846</th>\n",
       "      <th>1847</th>\n",
       "      <th>1848</th>\n",
       "      <th>1849</th>\n",
       "      <th>1850</th>\n",
       "      <th>1851</th>\n",
       "      <th>1852</th>\n",
       "      <th>1853</th>\n",
       "      <th>1854</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50057</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.6575</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32511</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5192</th>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1027</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12172</th>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.4449</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33003</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.4019</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1867 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       days_to_event  favorites  retweets  mention_count  hashtag_count  \\\n",
       "50057              2          2         2              0              2   \n",
       "32511              1          3         2              2              1   \n",
       "5192              38          1         0              1              2   \n",
       "12172             54          0         0              0              0   \n",
       "33003              1          1         0              0              1   \n",
       "\n",
       "       pos_emoji  neg_emoji  punc_excl  punc_ques  vaderTextScore  ...   1845  \\\n",
       "50057          0          0          0          0          0.6575  ...    0.0   \n",
       "32511          0          0          0          0          0.0000  ...    0.0   \n",
       "5192           1          0          0          0          0.1027  ...    0.0   \n",
       "12172          0          0          0          0         -0.4449  ...    0.0   \n",
       "33003          0          0          0          0         -0.4019  ...    0.0   \n",
       "\n",
       "       1846  1847  1848  1849  1850  1851  1852  1853  1854  \n",
       "50057   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "32511   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "5192    0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "12172   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "33003   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "\n",
       "[5 rows x 1867 columns]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_training_features_df.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Final processed data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_training_features_df.to_csv('./Datasets/social_unrest_processed.csv', sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Y = final_training_features_df['violent'] \n",
    "X = final_training_features_df.loc[:, final_training_features_df.columns != 'violent']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 60387 entries, 43492 to 33003\n",
      "Columns: 1866 entries, days_to_event to 1854\n",
      "dtypes: float64(1857), int64(9)\n",
      "memory usage: 860.2 MB\n"
     ]
    }
   ],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43492    0\n",
       "48760    0\n",
       "15962    1\n",
       "21152    1\n",
       "13258    1\n",
       "Name: violent, dtype: int64"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=np.random.seed(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(48309, 1866)\n",
      "(48309,)\n",
      "(12078, 1866)\n",
      "(12078,)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(Y_train.shape)\n",
    "print(X_test.shape)\n",
    "print(Y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Testing with imbalanced data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.506385973628\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(Y_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "seems like around 50.7% accuracy could be achieved by simply choosing majority class (not violent)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.924837938324\n"
     ]
    }
   ],
   "source": [
    "logistic_clf = LogisticRegression()\n",
    "logistic_score = cross_val_score(logistic_clf, X_train, Y_train, scoring='accuracy', cv=5)\n",
    "print(np.mean(logistic_score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our model is better than baseline as it is above 92% accurate."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Naive Bayes' Model/Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.871017655309\n"
     ]
    }
   ],
   "source": [
    "nb_clf = BernoulliNB()\n",
    "nb_score = cross_val_score(nb_clf, X_train, Y_train, scoring='accuracy', cv=5)\n",
    "print(np.mean(nb_score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our model is better than baseline as it gives around 87% accuracy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SVM Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.834171795129\n"
     ]
    }
   ],
   "source": [
    "svm_clf = SVC()\n",
    "svm_score = cross_val_score(svm_clf, X_train, Y_train, scoring='accuracy', cv=5)\n",
    "print(np.mean(svm_score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our model gives over 83% accuracy and is clearly better than baseline accuracy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Neural Network Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.930012998191\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "nn_clf = MLPClassifier()\n",
    "nn_score = cross_val_score(nn_clf, X_train, Y_train, scoring='accuracy', cv=5)\n",
    "print(np.mean(nn_score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our model is clearly better than baseline as it gives around 93% accuracy without tuning hyperparameters."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Final Accuracy Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(classifier, xTrain, yTrain, xTest, yTest):\n",
    "    classifier.fit(xTrain, yTrain)\n",
    "    yPred = classifier.predict(xTest)\n",
    "    accuracy = accuracy_score(yTest, yPred)\n",
    "    confusionMat = confusion_matrix(yTest, yPred)\n",
    "    f1 = f1_score(yTest, yPred)\n",
    "    return (accuracy, confusionMat, f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Accuracy prediction on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_logistic_clf = LogisticRegression()\n",
    "final_nb_clf = BernoulliNB()\n",
    "final_nn_clf = MLPClassifier()\n",
    "\n",
    "logistic_evaluation = evaluate(final_logistic_clf, X_train, Y_train, X_test, Y_test)\n",
    "nb_evaluation = evaluate(final_nb_clf, X_train, Y_train, X_test, Y_test)\n",
    "nn_evaluation = evaluate(final_nn_clf, X_train, Y_train, X_test, Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Accuracy:\n",
      "0.9258155323729094\n",
      "\n",
      "Logistic Regression F1 Score:\n",
      " 0.926048200726\n",
      "Logistic Regression Confusion Matrix:\n",
      "[[5572  333]\n",
      " [ 563 5610]]\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAD0CAYAAAB97VinAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAD99JREFUeJzt3W+MFdd9xvHvsxjbcZsaCCkigAuS\nt67Ii/jPCqjSF65RAbtVlxeJBanqrUXLGyKlUqUWV61I/UdK3tRN1MQSKsg4ao1RWgsUIdMVtmU1\nim2InbrGFHlLjIBio/VimgjFMc6vL+bcyyzZuzvrPdf33tnnI4125szszPHKPHNmzpkZRQRmZgB9\nna6AmXUPB4KZNTkQzKzJgWBmTQ4EM2tyIJhZkwPBzJocCGbW5EAws6ZrOl0BszpYoJvjAy5V2van\nnDsUERvaXKWPxIFglsEHXGJAf1Zp2+fjwYVtrs5H5kAwy0UVt+vix4ccCGYZCFBfxUT4RVurMiMO\nBLMcBKraQuhiDgSzXGqQCLO+21HSBkknJI1I2t7p+tSNpN2Szkt6vdN1aTep2tTNZnUgSJoDfAu4\nG1gJbJa0srO1qp3Hga7sYstLqK/a1M1mdSAAq4CRiDgZET8H9gKDHa5TrUTEC8BYp+vRdsKBUANL\ngNOl5TOpzGxaBEiqNHUz31Q0y6W7/61XMtsD4SywrLS8NJWZTY+mMQ6hi832S4YjQL+kFZKuBTYB\nBzpcJ+tR7mXocRFxGfgycAg4DuyLiGOdrVW9SHoS+AFwi6QzkrZ0uk5tU4NEmO2XDETEQeBgp+tR\nVxGxudN1+Lh0+b/1SmZ9IJhlIaE5vd/gdiCYZVB0O3a6FjPnQDDLpNvHGFThQDDLpffzYHb3MjRI\n2trpOtRd7f/GHrpcK/X+n7U71P9vrIpTF/Mlg1kmvofQwlzdENczrx27bovruJFP6jNd/Ka7X3bL\n7Z/pdBWm5aabbmLgjoGe+hu/deotRkdHq/0rl9AcB8KErmced+hP27FrSw6/9NVOV6H2Vq9eVXnb\nxtOOuUh6C/gJ8CFwOSIGJC0AngKWA28B90bEBRUH/gZwD3AJ+JOIeCXtZwj4m7TbhyNiz2TH9T0E\ns0zaMHL5dyPi1ogYSMvbgcMR0Q8cTstQvOCnP01bgceK+mgBsANYTfHujx2S5k92QAeCWS7tf5Zh\nEGic4fcAG0vlT0ThRWCepMXAemA4IsYi4gIwzBRvr3IgmOVQMQtSHiyUdLQ0TdQDE8C/S/phaf2i\niDiX5t8GFqX5Vi/6mfYLgNzLYJbJNMYYjJYuA1r5nYg4K+nXgWFJ/11eGREhKftNWrcQzDJofKgl\n18CkiDibfp4Hnqa4B/BOuhQg/TyfNm/1op9pvwDIgWCWQ8X3KVbpiZD0K5I+2ZgH1gGvU7y8Zyht\nNgTsT/MHgPtUWANcTJcWh4B1kuanm4nrUllLvmQwyyXf6XUR8HQKj2uAf4mIZyQdAfall8ycAu5N\n2x+k6HIcoeh2vB8gIsYkPUTxZjCAByNi0jdgOxDMMsk1DiEiTgKfm6D8XWDtBOUBbGuxr93A7qrH\ndiCYZeKhy2ZWEKgGd+QcCGa5uIVgZlBkQZ8fbjKzQve/Yr0KB4JZJjXIAweCWS7d/nq0KhwIZjnU\n5D3sDgSzTGqQBw4EsxwaDzf1OgeCWQ41+Ry8A8Esi+7/5kIVDgSzTHwPwcyuqEEiOBDMcvA9BDNr\nqMkwBAeCWTY1SAQHglkOftrRzK7w045mVlKDPHAgmGXhXgYzG6cGTQQHglkG7nY0s3F8yWBmBfnh\nJjMrcSCYWUH+cpOZlfV+HjgQzHLwK9TMbBxfMphZQQK3EMysoa8GgVDpA9aSNkg6IWlE0vZ2V8qs\n54iihVBl6mJTBoKkOcC3gLuBlcBmSSvbXTGzXiNVm6rvT3MkvSrpe2l5haSX0on5KUnXpvLr0vJI\nWr+8tI8HUvkJSeunOmaVFsIqYCQiTkbEz4G9wGD1/yyz+mv0MlSZpuErwPHS8teBRyPiZuACsCWV\nbwEupPJH03akE/cm4LPABuDb6QTfUpVAWAKcLi2fSWXjSNoq6aikox9wqcJuzWomYxNB0lLg94F/\nSssC7gK+mzbZA2xM84NpmbR+bdp+ENgbEe9HxI+BEYoTfEuV7iFUERE7I2IgIgbmckOu3Zr1BglV\nnICFjZNnmrZOsMd/AP4S+EVa/hTwXkRcTsvlE3PzpJ3WX0zbVzqZl1XpZTgLLCstL01lZlYyjXcq\njkbEQKuVkv4AOB8RP5R0Z466VVUlEI4A/ZJWUATBJuBLba2VWS/KNzDp88AfSroHuB74NeAbwDxJ\n16RWQPnE3Dhpn5F0DXAj8C4f4WQ+5SVDOviXgUMUNzj2RcSx6v9tZrOAQH3VpqlExAMRsTQillOc\ngJ+NiD8CngO+kDYbAvan+QNpmbT+2YiIVL4p9UKsAPqBlyc7dqWBSRFxEDhYZVuz2ah4Y1Lbxxj8\nFbBX0sPAq8CuVL4L+I6kEWCMIkSIiGOS9gFvAJeBbRHx4WQH8EhFs1zaMOgoIp4Hnk/zJ5mglyAi\nfgZ8scXvPwI8UvV4DgSzHK70IPQ0B4JZJjXIAweCWS6ak21YT8c4EMxyqMeX3BwIZtl0+ZOMVTgQ\nzDL4mLod286BYJaDv8tgZuP0fh44EMxy8SWDmRUEfe52NLOm3m8gOBDMcnAvg5mNU4M8cCCY5eJA\nMLOCn3Y0s7Ia5IEDwSwHUY9PuTkQzDLxJYOZFfz4s5mVqQYjkxwIZhkUA5M6XYuZcyCYZeJAMLMm\n31Q0s4JfkGJmZTVoIDgQzHLw045mNk4N8sCBYJaLWwhmVvBIRTNrENBXg0RwIJhlUoM8cCCY5eJ7\nCGbWVIM8oPdfJG/WDdIr1KpMU+9K10t6WdJ/Sjom6e9S+QpJL0kakfSUpGtT+XVpeSStX17a1wOp\n/ISk9VMd24FglkHjaccqUwXvA3dFxOeAW4ENktYAXwcejYibgQvAlrT9FuBCKn80bYeklcAm4LPA\nBuDbkuZMdmAHglkmuQIhCj9Ni3PTFMBdwHdT+R5gY5ofTMuk9WtVNEUGgb0R8X5E/BgYAVZNdmwH\nglkmfX2qNAELJR0tTVuv3pekOZJ+BJwHhoH/Ad6LiMtpkzPAkjS/BDgNkNZfBD5VLp/gdybUlpuK\nv3nbYp75/t+2Y9eWrJ371U5XofZO8L/VN9a0ehlGI2Jgsg0i4kPgVknzgKeB36pemY/OLQSzXFRx\nmoaIeA94DvhtYJ6kxkl8KXA2zZ8FlgGk9TcC75bLJ/idCTkQzDJoPO2YqZfh06llgKRPAL8HHKcI\nhi+kzYaA/Wn+QFomrX82IiKVb0q9ECuAfuDlyY7tcQhmmWQcmLQY2JN6BPqAfRHxPUlvAHslPQy8\nCuxK2+8CviNpBBij6FkgIo5J2ge8AVwGtqVLkZYcCGaZ5MqDiHgNuG2C8pNM0EsQET8DvthiX48A\nj1Q9tgPBLAd/29HMGvwpNzMbxy0EMyv4BSlmVuYWgpkBaRyC7yGYWUMNGggOBLMs3O1oZmXudjQz\nwF9uMrOr1CAPHAhmWTTeodbjHAhmmfiSwcyaapAHDgSzPOSBSWZWkNztaGYlvodgZk01yAMHglku\nbiGYGdD4KpMDwcySGuSBA8EsD9HX1/ufOXEgmGXiFoKZAekegschmFmDWwhm1qTpfsm1CzkQzHLp\n/TxwIJjl4nEIZgYUYeCHm8ysqQYNBAeCWS6+ZDCzphrkgQPBLIuaPNzU+4OvzbpA46XLVaYp9yUt\nk/ScpDckHZP0lVS+QNKwpDfTz/mpXJK+KWlE0muSbi/tayht/6akoamO7UAwyyRXIACXgb+IiJXA\nGmCbpJXAduBwRPQDh9MywN1Af5q2Ao8V9dECYAewGlgF7GiESCsOBLNM+qRK01Qi4lxEvJLmfwIc\nB5YAg8CetNkeYGOaHwSeiMKLwDxJi4H1wHBEjEXEBWAY2DDZsX0PwSyTdtxDkLQcuA14CVgUEefS\nqreBRWl+CXC69GtnUlmr8pambCFI2i3pvKTXK9TfbFaqermQMmOhpKOlaevE+9SvAv8K/HlE/F95\nXUQEELn/O6q0EB4H/hF4IvfBzepjWp+DH42IgUn3Js2lCIN/joh/S8XvSFocEefSJcH5VH4WWFb6\n9aWp7Cxw51Xlz0923ClbCBHxAjA21XZms13GXgYBu4DjEfH3pVUHgEZPwRCwv1R+X+ptWANcTJcW\nh4B1kuanm4nrUllL2e4hpGbPVoBly27KtVuznpHxHsLngT8G/kvSj1LZXwNfA/ZJ2gKcAu5N6w4C\n9wAjwCXgfoCIGJP0EHAkbfdgREx6cs8WCBGxE9gJcMftd2S/tjHrepnyICL+Y5K9rZ1g+wC2tdjX\nbmB31WO7l8EsA3/KzczGmRVDlyU9CfwAuEXSmXT9YmZXUcWpm03ZQoiIzR9HRcx6XR1aCL5kMMuk\nBnngQDDLQZrWwKSu5UAwy6QGeeBAMMvF3Y5m1uQWgpkBjecUej8R/IIUM2tyC8Eskzq0EBwIZpnU\nIA8cCGa5OBDMrMmfgzezK3o/DxwIZjlM45sLXc2BYJaFfMlgZiW9nwcOBLNcapAHDgSzXOSHm8ys\noffjwIFglkXxOfjejwQ/3GRmTW4hmOXgcQhmVuZLBjOrFbcQzLIQfTVoITgQzHLp/TxwIJjlUHQ7\ndroWM+dAMMukBnngQDDLoiZNBAeCWSa9HwcOBLNsatBA8DgEs1waH3ydaqq4r92Szkt6vVS2QNKw\npDfTz/mpXJK+KWlE0muSbi/9zlDa/k1JQ1Md14FglkHjFkKVqaLHgQ1XlW0HDkdEP3A4LQPcDfSn\naSvwGBQBAuwAVgOrgB2NEGnFgWCWjSpOU4uIF4Cxq4oHgT1pfg+wsVT+RBReBOZJWgysB4YjYiwi\nLgDD/HLIjON7CGaZTOPsv1DS0dLyzojYWeH3FkXEuTT/NrAozS8BTpe2O5PKWpW35EAw+/iNRsTA\nTHYQESEpclWowZcMZjlUvH8ww56Id9KlAOnn+VR+FlhW2m5pKmtV3pIDwSybfPcQWjgANHoKhoD9\npfL7Um/DGuBiurQ4BKyTND/dTFyXylryJYNZJjnHIUh6EriT4n7DGYregq8B+yRtAU4B96bNDwL3\nACPAJeB+gIgYk/QQcCRt92BEXH2jchwHglkuGQMhIja3WLV2gm0D2NZiP7uB3VWP25ZAeOXVV0Y/\nccO1p9qx7zZZCIx2uhI114t/49+ouqH85abWIuLT7dhvu0g6OtO7vjY5/417gy8ZzDLxswxmVitu\nIRSqjBKzman331j1eOuyAwGoOGzUZmBW/I17Pw8cCGa51CAPHAhm2dTgksE3Fc2syS0Es0x6v33g\nQDDLpwaJ4EAwy6B4hVrvJ4IDwSyT3o8DB4JZHjN+1UF3cCCYZdP7ieBAMMuk9+PAgWCWTw0SwYFg\nlkkN8sCBYJbHzF+p3A0cCGaZ1CAP/CyDmV3hFoJZBnUZqajiDc5mNhOSnqF4s3QVoxEx6UdXO8WB\nYGZNvodgZk0OBDNrciCYWZMDwcyaHAhm1uRAMLMmB4KZNTkQzKzJgWBmTf8PwmWB8sDD5MEAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13a2c0128>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Logistic regression accuracy\n",
    "print(\"Logistic Regression Accuracy:\\n{}\\n\".format(logistic_evaluation[0]))\n",
    "print(\"Logistic Regression F1 Score:\\n\",logistic_evaluation[2])\n",
    "\n",
    "print(\"Logistic Regression Confusion Matrix:\\n{}\\n\".format(logistic_evaluation[1]))\n",
    "plt.matshow(logistic_evaluation[1], cmap='Purples')\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Logistic Regression Model gives 92.5% accuracy with f1 score of 0.926"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Neural Network Accuracy:\n",
      "0.930534856764365\n",
      "\n",
      "Neural Network F1 score:\n",
      " 0.931526972986\n",
      "Neural Network Confusion Matrix:\n",
      "[[5532  373]\n",
      " [ 466 5707]]\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAD0CAYAAAB97VinAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEAxJREFUeJzt3W+MFdd9xvHvswuYyI0NhJZS/hQk\nb1qRFw3OClylL1yjAnarwovEwarqlYVKXxApVSu1OKpEa8dS8qZuLaWuUEHGUWuCklqgiJpusYmV\nKrbBf+oYU8SWGAHFRuvFtC6KY9JfX8y517N47+6s96zvvbPPRxrtzLmzM2dX8OyZc2bOKCIwMwPo\naXcFzKxzOBDMrMmBYGZNDgQza3IgmFmTA8HMmhwIZtbkQDCzJgeCmTXNancFzOpggW6J97laad93\nuXg4IjZOc5U+EgeCWQbvc5V+/UGlfY/GAwunuTofmQPBLBdV3K+DHx9yIJhlIEA9FRPh/6a1KlPi\nQDDLQaCqLYQO5kAwy6UGiTDjhx0lbZR0StKQpB3trk/dSNoj6ZKk19pdl+kmVVs62YwOBEm9wDeB\nO4FVwD2SVrW3VrXzGNCRQ2x5CfVUWzrZjA4EYA0wFBFnIuKnwD5gU5vrVCsR8Sww0u56TDvhQKiB\nJcC50vb5VGY2KQIkVVo6mTsVzXLp7P/rlcz0QLgALCttL01lZpOjSdyH0MFm+iXDMaBP0kpJc4At\nwME218m6lEcZulxEXAO+DBwGTgL7I+JEe2tVL5KeAH4I/Iqk85K2trtO06YGiTDTLxmIiEPAoXbX\no64i4p521+Hj0uH/1yuZ8YFgloWEeru/we1AMMugGHZsdy2mzoFglkmn32NQhQPBLJfuz4OZPcrQ\nIGlbu+tQd7X/HfvW5Vqp9z/WzlD/37EqLh3MlwxmmbgPoYU5ujHmat50HHpazNXN3NSzpINnuvuw\nT69e3O4qTMry5cvp/1x/V/2O3zj7BsPDw9X+l0uo14Ewprmax9q5fzgdh7bkqef/vN1VqL21a9dU\n3rfxtGO38yWDWSY1yAMHglk2NUgEjzKY5VDxuaaqmSHpDUk/kvSKpOOpbIGkQUmn09f5qVySHknz\ngr4q6dbScQbS/qclDUx0XgeCWSbTcB/Cb0bEZyOiP23vAI5ERB9wJG1DMSdoX1q2AY9CESDATmAt\nxXSBOxsh0ooDwSyDxotapvnGpE3A3rS+F9hcKn88Cs8B8yQtBjYAgxExEhGXgUEmmPDWgWCWQ8X5\nFCcxEhHAv0h6sXSX56KIuJjW3wQWpfVWc4NOes5Qdyqa5VL9z+vCRr9Asisidl23z29ExAVJvwAM\nSvqP8ocREZKy39fhQDDLZBJ//YdL/QJjiogL6eslSU9S9AG8JWlxRFxMlwSX0u6t5ga9ANx+XfnR\n8c7rSwazTHJdMki6UdInG+vAeuA1ivk+GyMFA8CBtH4QuDeNNtwGXEmXFoeB9ZLmp87E9amsJbcQ\nzHIQKN+f10XAkyk8ZgH/GBFPSToG7E/zUp4F7k77HwLuAoaAq8B9ABExIulBismEAR6IiHFfmuNA\nMMsl041JEXEG+LUxyt8G1o1RHsD2FsfaA+ypem4HglkGEvT44SYzK3T+FOtVOBDMMqlBHjgQzHLp\n9OnRqnAgmOVQk3nYHQhmmdQgDxwIZjk0Hm7qdg4Esxxq8jp4B4JZFp3/zoUqHAhmmbgPwcw+UINE\ncCCY5eA+BDNrqMltCA4Es2xqkAgOBLMc/LSjmX3ATzuaWUkN8sCBYJaFRxnMbJQaNBEcCGYZeNjR\nzEbxJYOZFeSHm8ysxIFgZgVN6lVuHcuBYJZL9+eBA8EsB0+hZmaj+JLBzAoSuIVgZg09NQiESi+w\nlrRR0ilJQ5J2THelzLqOKFoIVZYONmEgSOoFvgncCawC7pG0arorZtZtpGpLJ6vSQlgDDEXEmYj4\nKbAP2DS91TLrLo1RhipLJ6sSCEuAc6Xt86lsFEnbJB2XdPx9/jdX/cy6Rw2aCJX6EKqIiF0R0R8R\n/bO5MddhzbqDhCou1Q+pXkkvS/pe2l4p6fnUl/dtSXNS+Q1peyh9vqJ0jPtT+SlJGyY6Z5VAuAAs\nK20vTWVmVtLTq0rLJHwFOFna/gbwcETcAlwGtqbyrcDlVP5w2o/U17cF+AywEfjb1CfY+meoUKlj\nQF9KpznpBAcr/0hmM0XGSwZJS4HfBv4+bQu4A/hO2mUvsDmtb0rbpM/Xpf03Afsi4r2I+DEwRNEn\n2NKE9yFExDVJXwYOA73Anog4UemnMpspBMp2AQ7AXwN/CnwybX8KeCcirqXtcl9es58v/X+9kvZf\nAjxXOuaY/X9llW5MiohDwKEq+5rNRMWMSZUvBxZKOl7a3hURu5rHkn4HuBQRL0q6PV8tJ+Y7Fc1y\nqT6kOBwR/eN8/nngdyXdBcwFbgL+BpgnaVZqJZT78hr9fOclzQJuBt7mI/T/5W3kmM1UGUcZIuL+\niFgaESso+uyejojfA54BvpB2GwAOpPWDaZv0+dMREal8SxqFWAn0AS+Md263EMwy+RhuMfgzYJ+k\nrwEvA7tT+W7gW5KGgBGKECEiTkjaD7wOXAO2R8TPxjuBA8EsE/Xmb3BHxFHgaFo/wxijBBHxE+CL\nLb7/IeChqudzIJjl0Pk3IVbiQDDLpcOfU6jCgWCWwSSHHTuWA8EsB7+XwcxG6f48cCCY5eJLBjMr\nCHqmYdjx4+ZAMMul+xsIDgSzHDzKYGaj1CAPHAhmuTgQzKwwyfkSO5UDwSyTGuSBA8EsB1GPV7k5\nEMwy8SWDmRX8+LOZlakGdyY5EMwyKG5Manctps6BYJaJA8HMmtypaGYFT5BiZmU1aCA4EMxy8NOO\nZjZKDfLAgWCWi1sIZlbwnYpm1iCgpwaJ4EAwy6QGeeBAMMvFfQhm1lSDPHAgmGXhKdTMrMFPO5rZ\nKHUIhO5/95RZh+jpUaVlIpLmSnpB0r9LOiHpL1P5SknPSxqS9G1Jc1L5DWl7KH2+onSs+1P5KUkb\nJjr3tLQQ+lb/Iod++NXpOLQl62b/RburUHun+K/qOyvrKMN7wB0R8a6k2cAPJP0z8MfAwxGxT9Lf\nAVuBR9PXyxFxi6QtwDeAL0laBWwBPgP8EvCvkj4dET9rdWK3EMxyUcVlAlF4N23OTksAdwDfSeV7\ngc1pfVPaJn2+TkU6bQL2RcR7EfFjYAhYM965HQhmGTSedqyyAAslHS8t2z50PKlX0ivAJWAQ+E/g\nnYi4lnY5DyxJ60uAcwDp8yvAp8rlY3zPmNypaJbJJC4ZhiOif7wdUrP+s5LmAU8CvzrF6lXiFoJZ\nJlK1ZTIi4h3gGeDXgXmSGn/ElwIX0voFYFlRB80CbgbeLpeP8T1jciCY5VDxcqFKK0LSz6eWAZI+\nAfwWcJIiGL6QdhsADqT1g2mb9PnTERGpfEsahVgJ9AEvjHduXzKYZZD5VW6Lgb2Sein+aO+PiO9J\neh3YJ+lrwMvA7rT/buBbkoaAEYqRBSLihKT9wOvANWD7eCMM4EAwyybXsGNEvAqsHqP8DGOMEkTE\nT4AvtjjWQ8BDVc/tQDDLwROkmFmZH24yMyDdh+D3MphZQw0aCA4Esyw8H4KZlWUcdmwbB4JZBn5z\nk5mNUoM8cCCYZVGTOdQcCGaZ+JLBzJpqkAcOBLM85BuTzKwgedjRzErch2BmTTXIAweCWS5uIZgZ\n0Jgv0YFgZkkN8sCBYJaH6Onp/jmLHQhmmbiFYGZA6kPwfQhm1uAWgpk1qcqbXDucA8Esl+7PAweC\nWS6+D8HMgCIM/HCTmTXVoIHgQDDLxZcMZtZUgzxwIJhl4YebzKyhJpMuOxDMcnEgmFlTTw0Sofuf\n1zTrEEovfJ1oqXCcZZKekfS6pBOSvpLKF0galHQ6fZ2fyiXpEUlDkl6VdGvpWANp/9OSBiY694SB\nIGmPpEuSXpvwJzGboYoZk6otFVwD/iQiVgG3AdslrQJ2AEciog84krYB7gT60rINeLSokxYAO4G1\nwBpgZyNEWqnSQngM2FjpxzCbsaq1Dqq0ECLiYkS8lNb/BzgJLAE2AXvTbnuBzWl9E/B4FJ4D5kla\nDGwABiNiJCIuA4NM8H95wj6EiHhW0ooJfwqzGW4SXQgLJR0vbe+KiF1jH1MrgNXA88CiiLiYPnoT\nWJTWlwDnSt92PpW1Km8pW6eipG0UzRWWL1+e67BmXWMS9yEMR0R/heP9HPBd4I8i4r/Lx4+IkBQf\nqaLjyNapGBG7IqI/IvoXLlyY67Bm3UMVlyqHkmZThME/RMQ/peK30qUA6eulVH4BWFb69qWprFV5\nSx5lMMug8Sq3KsvEx5KA3cDJiPir0kcHgcZIwQBwoFR+bxptuA24ki4tDgPrJc1PnYnrU1lLvg/B\nLJOMty5/Hvh94EeSXkllXwW+DuyXtBU4C9ydPjsE3AUMAVeB+wAiYkTSg8CxtN8DETEy3oknDARJ\nTwC3U3SEnAd2RsTu6j+b2cyQKw4i4gfjHG7dGPsHsL3FsfYAe6qeu8oowz1VD2Y2k/nhJjNrqkEe\nOBDMcqh601GncyCYZVKDPHAgmOXiSVbNrMktBDMDGk8ydn8i+E5FM2tyC8Eskzq0EBwIZpnUIA8c\nCGa5OBDMrMmvgzezD3R/HjgQzHKYxASqHc2BYJaFfMlgZiXdnwcOBLNcapAHDgSzXOSHm8ysofvj\nwIFglkXxOvjujwQ/3GRmTW4hmOXg+xDMrMyXDGZWK24hmGUhemrQQnAgmOXS/XngQDDLoRh2bHct\nps6BYJZJDfLAgWCWRU2aCA4Es0y6Pw4cCGbZ1KCB4EAwy6UONyY5EMwyqEkXgu9UNMtHFZcKR5L2\nSLok6bVS2QJJg5JOp6/zU7kkPSJpSNKrkm4tfc9A2v+0pIGJzutAMMukMdHqREtFjwEbryvbARyJ\niD7gSNoGuBPoS8s24NGiPloA7ATWAmuAnY0QacWBYNaBIuJZYOS64k3A3rS+F9hcKn88Cs8B8yQt\nBjYAgxExEhGXgUE+HDKjuA/BLIfJ/fVfKOl4aXtXROyq8H2LIuJiWn8TWJTWlwDnSvudT2Wtylty\nIJhlUzkRhiOifypnioiQFFM5xlh8yWCWSeY+hLG8lS4FSF8vpfILwLLSfktTWavylhwIZrnkG2Ro\n5SDQGCkYAA6Uyu9Now23AVfSpcVhYL2k+akzcX0qa2laLhleeuml4Tk3zD47HceeJguB4XZXoua6\n8Xf8y1V3VOY3N0l6Aridor/hPMVowdeB/ZK2AmeBu9Puh4C7gCHgKnAfQESMSHoQOJb2eyAiru+o\nHH3eiOyXIV1H0vGpXtPZ+Or+O7519efi+0f/rdK+N837xIud+rtwp6JZJr5T0cxqxS2EQpUxYJua\nev+O5YebaqPiTSE2BTPid9z9eeBAMMulBnngQDDLpgaXDO5UNLMmtxDMMun+9oEDwSyfGiSCA8Es\ng2IKte5PBAeCWSbdHwcOBLM8pv4kY0dwIJhl0/2J4EAwy6T748CBYJZPDRLBgWCWSQ3ywIFglsfU\nJ0zsBA4Es0xqkAd+lsHMPuAWglkGdblT0ZOsmmUg6SmKmaWrGI6IcV+p1i4OBDNrch+CmTU5EMys\nyYFgZk0OBDNrciCYWZMDwcyaHAhm1uRAMLMmB4KZNf0//DeBKkJ0eHEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13a52add8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Neural Network accuracy\n",
    "print(\"Neural Network Accuracy:\\n{}\\n\".format(nn_evaluation[0]))\n",
    "print(\"Neural Network F1 score:\\n\",nn_evaluation[2])\n",
    "\n",
    "print(\"Neural Network Confusion Matrix:\\n{}\\n\".format(nn_evaluation[1]))\n",
    "plt.matshow(nn_evaluation[1], cmap='Purples')\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Neural Network gives 93% accuracy with f1 score of 0.931"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Naive Bayes' Accuracy:\n",
      "0.8692664348402054\n",
      "\n",
      "Naive Bayes' F1 Score: 0.862396514161\n",
      "Naive Bayes' Confusion Matrix:\n",
      "[[5551  354]\n",
      " [1225 4948]]\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAD0CAYAAAB97VinAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAD+JJREFUeJzt3W2MFdd9x/Hvb8EPfXANhJRiwDWV\nkSMi1Q9BQJW+cI2KsVsVv0gsaFWvLFreECmVWjW4b2jtWEre1KnVxCoqyDhqjVFSCxQh0xU2sirF\nNrR2iTG12BAjdoWN1otpIxTHJP++mHMvs2Tv7lw4l3vv7O8jjZg599wzsyvvz2fOmQdFBGZmAAPd\nPgAz6x0OBDNrciCYWZMDwcyaHAhm1uRAMLMmB4KZNTkQzKzJgWBmTbO7fQBmdTBPt8cnXKhU98ec\nORAR6zp8SFfEgWCWwSdcYIX+vFLdQ/H4/A4fzhVzIJjloor1evj2IQeCWQYCNFAxEX7e0UO5Kg4E\nsxwEqtpD6GEOBLNcapAIM37aUdI6Se9KGpa0tdvHUzeSdko6K+ntbh9Lp0nVll42owNB0izgm8AD\nwHJgo6Tl3T2q2nkW6MkptryEBqotvWxGBwKwEhiOiJMR8VNgN7C+y8dUKxHxKjDe7ePoOOFAqIFF\nwOnS9kgqM2uLAEmVll7mQUWzXHr7b72SmR4Io8CS0vbiVGbWHrVxHUIPm+mnDIeBZZKWSroe2ADs\n6/IxWZ/yLEOfi4iLwJeAA8BxYE9EHOvuUdWLpOeB7wN3SBqRtKnbx9QxNUiEmX7KQETsB/Z3+zjq\nKiI2dvsYrpUe/1uvZMYHglkWEprV/x1uB4JZBsW0Y7eP4uo5EMwy6fVrDKpwIJjl0v95MLNnGRok\nbe72MdRd7X/HvnS5Vur9H2tvqP/vWBWXKk1J70n6gaS3JB1JZfMkDUk6kf6dm8ol6el0x+5RSfeU\n2hlM9U9IGpxuvw4Es0w6cC/D70XEXRGxIm1vBQ5GxDLgYNqG4m7dZWnZDDyTjmcesA1YRXEj37ZG\niLTSkTGE6/TLcSNzOtF0R9zAzdykW3r4SXe/6I57bun2IbTl1ltvZcXnVvTV7/i9U+8xNjZW7S9Y\nQrM6fjqwHrg3re8CDgFfSeXPRUQAr0maI2lhqjsUEePFIWqI4lb051vtoCOBcCNz+Bx/1ommLTn4\n+t92+xBqb9WqlZXrNu52zCiAf5cUwD9FxHZgQUScSZ+/DyxI663u2m37bl7PMphl0kYezG+MCyTb\n0x982e9GxKikXweGJP1P+cOIiBQWWTkQzHKpnghjpXGBSUXEaPr3rKQXKcYAPpC0MCLOpFOCs6l6\nq7t2R7l0itEoPzTVfj2oaJZDxfuaqmSGpF+RdFNjHVgLvE1xJ25jpmAQ2JvW9wGPpNmG1cD5dGpx\nAFgraW4aTFybylpyD8Esk4zXGCwAXkxjErOBf42IlyQdBvakO0ZPAQ+n+vuBB4Fh4ALwKEBEjEt6\nguI2f4DHGwOMrTgQzDJo60Ut04iIk8Cdk5R/CKyZpDyALS3a2gnsrLpvB4JZDn3wvMQqHAhmudRg\nRM6BYJaJewhm1uRAMLOCQD5lMLMm9xDMDIosGOj8zU0d50Awy6L3H7FehQPBLJMa5IEDwSyXXn88\nWhUOBLMcavIcdgeCWSY1yAMHglkOOW9u6iYHglkONXkdvAPBLIvef+dCFQ4Es0w8hmBml9QgERwI\nZjl4DMHMGmpyGYIDwSybGiSCA8EsB9/taGaX+G5HMyupQR44EMyy8CyDmU1Qgy6CA8EsA087mtkE\nPmUws4J8c5OZlTgQzKwgv7nJzMr6Pw8cCGY5+BFqZjaBTxnMrCBBDXoINXhfrVlvGBhQpaUqSbMk\nvSnpe2l7qaTXJQ1LekHS9an8hrQ9nD6/rdTGY6n8XUn3T/szVDywdanBYUlbK/9EZjOFKHoIVZbq\nvgwcL21/HXgqIm4HzgGbUvkm4FwqfyrVQ9JyYAPwWWAd8C1Js6ba4bSBkBr4JvAAsBzYmHZkZiVS\ntaVaW1oM/AHwz2lbwH3Ad1KVXcBDaX192iZ9vibVXw/sjoiPI+JHwDCwcqr9VukhrASGI+JkRPwU\n2J12ZGZJY5ahylLRN4C/Bn6etj8FfBQRF9P2CLAorS8CTgOkz8+n+s3ySb4zqSqBUKlRSZslHZF0\n5BMuVGjWrGaqdxHmN/5W0rJ5YjP6Q+BsRPzntf4Rss0yRMR2YDvATbolcrVr1hekdqYdxyJixRSf\nfx74I0kPAjcCvwb8AzBH0uzUC1gMjKb6o8ASYETSbOBm4MNSeUP5O5Oq0kNou1GzmWhgliot04mI\nxyJicUTcRjEo+HJE/AnwCvCFVG0Q2JvW96Vt0ucvR0Sk8g1pFmIpsAx4Y6p9V+khHAaWpQZH0wH+\ncYXvmc0snb8w6SvAbklfBd4EdqTyHcC3JQ0D4xR/o0TEMUl7gHeAi8CWiPjZVDuYNhAi4qKkLwEH\ngFnAzog4doU/kFk9CdSBq3oi4hBwKK2fZJJZgoj4CfDFFt9/Eniy6v4qjSFExH5gf9VGzWaa4olJ\n/X+loi9dNsulBpcuOxDMcmhvlqFnORDMMqlBHjgQzHLRrP6/V9CBYJZDPd7k5kAwy8aDimYGnnY0\nszK/l8HMJuj/PHAgmOXiUwYzKwgGPO1oZk3930FwIJjl4FkGM5ugBnngQDDLxYFgZgXf7WhmZTXI\nAweCWQ6Ctl7T1qscCGaZ+JTBzAq+/dnMylSDK5McCGYZFBcmdfsorp4DwSwTB4KZNXlQ0cwKfkCK\nmZXVoIPgQDDLwXc7mtkENcgDB4JZLu4hmFnBVyqaWYOAgRokggPBLJMa5IEDwSyXOowh9P9zo816\nhFRtmb4d3SjpDUn/LemYpL9L5UslvS5pWNILkq5P5Tek7eH0+W2lth5L5e9Kun+6fTsQzHJIj1Cr\nslTwMXBfRNwJ3AWsk7Qa+DrwVETcDpwDNqX6m4BzqfypVA9Jy4ENwGeBdcC3JM2aascOBLMMGnc7\n5ughROHHafO6tARwH/CdVL4LeCitr0/bpM/XqEie9cDuiPg4In4EDAMrp9q3A8Esk1yBULSlWZLe\nAs4CQ8APgY8i4mKqMgIsSuuLgNMA6fPzwKfK5ZN8Z1IeVDTLpI1nKs6XdKS0vT0itpcrRMTPgLsk\nzQFeBD6T5yin1pFA+K3fXsALL/1VJ5q25OHPfKPbh1B7Pxz5oHpltTXLMBYRK6pUjIiPJL0C/A4w\nR9Ls1AtYDIymaqPAEmBE0mzgZuDDUnlD+TuT8imDWS6quEzXjPTp1DNA0i8Bvw8cB14BvpCqDQJ7\n0/q+tE36/OWIiFS+Ic1CLAWWAW9MtW+fMphlkPlux4XArjQjMADsiYjvSXoH2C3pq8CbwI5Ufwfw\nbUnDwDjFzAIRcUzSHuAd4CKwJZ2KtORAMMskVyBExFHg7knKTzLJLEFE/AT4You2ngSerLpvB4JZ\nJjW4UNGBYJaF3+1oZg1+lZuZTeAegpkV/IAUMytzD8HMgHQdgscQzKyhBh0EB4JZFp52NLMyTzua\nGeA3N5nZZWqQBw4Esywaz1Drcw4Es0x8ymBmTTXIAweCWR7yhUlmVpA87WhmJR5DMLOmGuSBA8Es\nF/cQzAxovJXJgWBmSQ3ywIFglocYGOj/9x45EMwycQ/BzIA0huDrEMyswT0EM2tSlTe59jgHglku\n/Z8HDgSzXHwdgpkBRRj45iYza6pBB8GBYJaLTxnMrKkGeeBAMMuiJjc39f/F12Y9oPHQ5SrLtG1J\nSyS9IukdScckfTmVz5M0JOlE+nduKpekpyUNSzoq6Z5SW4Op/glJg9Pt24FglkmuQAAuAn8ZEcuB\n1cAWScuBrcDBiFgGHEzbAA8Ay9KyGXimOB7NA7YBq4CVwLZGiLTiQDDLZECqtEwnIs5ExH+l9f8D\njgOLgPXArlRtF/BQWl8PPBeF14A5khYC9wNDETEeEeeAIWDdVPv2GIJZJp0YQ5B0G3A38DqwICLO\npI/eBxak9UXA6dLXRlJZq/KWpu0hSNop6ayktyscv9mMVPV0IWXGfElHSsvmydvUrwLfBf4iIv63\n/FlEBBC5f44qPYRngX8Ensu9c7P6aOt18GMRsWLK1qTrKMLgXyLi31LxB5IWRsSZdEpwNpWPAktK\nX1+cykaBey8rPzTVfqftIUTEq8D4dPXMZrqMswwCdgDHI+LvSx/tAxozBYPA3lL5I2m2YTVwPp1a\nHADWSpqbBhPXprKWso0hpG7PZoDFi5ZMU9usfjKOIXwe+FPgB5LeSmV/A3wN2CNpE3AKeDh9th94\nEBgGLgCPAkTEuKQngMOp3uMRMeX/3LMFQkRsB7YD3Hnn3dnPbcx6XqY8iIj/mKK1NZPUD2BLi7Z2\nAjur7tuzDGYZ+FVuZjbBjLh0WdLzwPeBOySNpPMXM7uMKi69bNoeQkRsvBYHYtbv6tBD8CmDWSY1\nyAMHglkOUlsXJvUsB4JZJjXIAweCWS6edjSzJvcQzAxo3KfQ/4ngB6SYWZN7CGaZ1KGH4EAwy6QG\neeBAMMvFgWBmTX4dvJld0v954EAwy6GNdy70NAeCWRbyKYOZlfR/HjgQzHKpQR44EMxykW9uMrOG\n/o8DB4JZFsXr4Ps/Enxzk5k1uYdgloOvQzCzMp8ymFmtuIdgloUYqEEPwYFglkv/54EDwSyHYtqx\n20dx9RwIZpnUIA8cCGZZ1KSL4EAwy6T/48CBYJZNDToIvg7BLJfGC1+nWyq2tVPSWUlvl8rmSRqS\ndCL9OzeVS9LTkoYlHZV0T+k7g6n+CUmD0+3XgWCWQWMIocpS0bPAusvKtgIHI2IZcDBtAzwALEvL\nZuAZKAIE2AasAlYC2xoh0ooDwSwbVVymFxGvAuOXFa8HdqX1XcBDpfLnovAaMEfSQuB+YCgixiPi\nHDDEL4bMBB5DMMvkGowhLIiIM2n9fWBBWl8EnC7VG0llrcpbciCYXXvzJR0pbW+PiO3tNBARISky\nH5cDwSyL9sYHxiJixRXs5QNJCyPiTDolOJvKR4ElpXqLU9kocO9l5Yem2oHHEMyyyTeG0MI+oDFT\nMAjsLZU/kmYbVgPn06nFAWCtpLlpMHFtKmvJPQSzTHKOIUh6nuL/7vMljVDMFnwN2CNpE3AKeDhV\n3w88CAwDF4BHASJiXNITwOFU7/GIuHygcgIHglkuGQMhIja2+GjNJHUD2NKinZ3Azqr77UggHD36\n1thv3HLzqU603SHzgbFuH0TN9ePv+DerVpTf3NRaRHy6E+12iqQjVzjIYxX5d9wffMpglonvZTCz\nWnEPodDWRSF2Rer9O1Y9nrrsQADavUrM2jcjfsf9nwcOBLNcapAHDgSzbGpwyuBBRTNrcg/BLJP+\n7x84EMzyqUEiOBDMMigeodb/ieBAMMuk/+PAgWCWx1U/6qA3OBDMsun/RHAgmGXS/3HgQDDLpwaJ\n4EAwy6QGeeBAMMujvccu9yoHglkmNcgD38tgZpe4h2CWQV2uVFTxBGczuxqSXqJ4snQVYxEx5UtX\nu8WBYGZNHkMwsyYHgpk1ORDMrMmBYGZNDgQza3IgmFmTA8HMmhwIZtbkQDCzpv8H/t9/BhKiTicA\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13a694320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Naive Bayes' accuracy\n",
    "print(\"Naive Bayes' Accuracy:\\n{}\\n\".format(nb_evaluation[0]))\n",
    "print(\"Naive Bayes' F1 Score:\",nb_evaluation[2])\n",
    "\n",
    "print(\"Naive Bayes' Confusion Matrix:\\n{}\\n\".format(nb_evaluation[1]))\n",
    "plt.matshow(nb_evaluation[1], cmap='Purples')\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Naive Bayes' Model gives around 87% accuracy with f1 score of 0.862"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_svm_clf = SVC()\n",
    "svm_evaluation = evaluate(final_svm_clf, X_train, Y_train, X_test, Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVM Accuracy:\n",
      "0.8372247060771651\n",
      "\n",
      "SVM F1 Score: 0.834706574744\n",
      "SVM Confusion Matrix:\n",
      "[[5148  757]\n",
      " [1209 4964]]\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAD0CAYAAAB97VinAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAE8ZJREFUeJzt3X+s1fV9x/Hn614V280JqGM6bSWV\nueCSMmXg0v3hMANky7BJa2A/vDFstgkmXbJ1hf2j05K0STs2EzWhg4JdV0rcjDeGld2hxnTxd2VW\nsI47xQyCErzIakiduPf++H7O4Xg9P74XPodzzve+Hsk395zP+Z7v98sNvPh8P9/v9/NWRGBmBjDU\n6wMws/7hQDCzOgeCmdU5EMyszoFgZnUOBDOrcyCYWZ0DwczqHAhmVndOrw/ArApm66p4nxOl1n2X\nw7siYnmXD+m0OBDMMnifEyzUn5Za94m4++IuH85pcyCY5aKS6/Xx40MOBLMMBGioZCL8X1cP5Yx4\nUNEsB4FKLqU2Jx2Q9GNJeyQ9n9pmSxqTtD/9nJXaJeleSeOSXpJ0bcN2RtL6+yWNdNqvA8Esl5yJ\nUPjtiFgQEQvT+3XA7oiYB+xO7wFuAual5XbggeJwNBu4E1gMLALurIVIK9M+ECQtl/RqStd1nb9h\nUyFpi6Qjkl7u9bF0W/48+IiVwLb0ehtwc0P7g1F4Gpgp6VJgGTAWERMRcQwYA9pe3ZjWgSBpGLiP\nImHnA6slze/tUVXOVjr8JawGoaFyS0kB/KukFyTdntrmRMTh9PpNYE56/cvAfzd892Bqa9Xe0nQf\nVFwEjEfEawCStlOk7b6eHlWFRMSTkq7s9XF0naYwqAgX18YFkk0RsWnSOr8VEYck/SIwJuknjR9G\nREjKfr1iugdCswRd3KNjsQEmQOXPB442jAs0FRGH0s8jkh6m+M/rLUmXRsThdEpwJK1+CLii4euX\np7ZDwA2T2p9ot99pfcpglpVKLp02I/2cpAtqr4GlwMvAKFC7UjACPJJejwK3pqsN1wPH06nFLmCp\npFlpMHFpamtpuvcQWiWr2dRM7ZShkznAw6nHcQ7wjxHxA0nPATskrQHeAG5J6+8EVgDjwAngNoCI\nmJB0D/BcWu/uiJhot+PpHgjPAfMkzaUIglXAH/T2kGxQneEVhLo0pvXpJu1vAzc2aQ9gbYttbQG2\nlN33tD5liIiTwB0U3ahXgB0Rsbe3R1Utkr4HPAVcLelg+t+tms7Cdcdum+49BCJiJ0WXy7ogIlb3\n+hjOlj7/t17KtA8EsywkNDz4HW4HglkGxWXHXh/FmXMgmGUyhfsQ+pYDwSyXwc+D6X2VoabhXnHr\nksr/jtN9CBmfZegJB0Kh2n9Z+0P1f8eZ7lTsJZ8ymGXiMYQWztXH43xmdmPTXTGDC7lAl/XxTHcf\ndfW1l/X6EKbkE5/4BAuvWzhQv+MDbxzg6NGj5f6VS2jYgdDU+czkOv6kG5u2ZPczd/X6ECpv8eJF\npded4tOOfcunDGaZVCAPHAhm2VQgERwIZjn0/3NLpTgQzDLp93sMynAgmGUwpUItfcyBYJaDVImr\nDL5T0SyXoZJLSZKGJb0o6dH0fquk11M1pz2SFqT2bJWb3EMwy6QLPYQvUczk9QsNbV+OiIcmrddY\nuWkxReWmxQ2VmxZS1Hl4QdJoKtrSlHsIZpkonTZ0Wkpu63Lgd4G/L7G6KzeZ9RWBhsotpEItDUuz\nB7/+FvhLPlorekM6LdgoaUZqc+Ums76TqVCLpN8DjkTEC5JuaPhoPUUJt/OATcBXgLtP72Cbcw/B\nLAMJhoZVainhM8DvSzoAbAeWSPqHiDicTgveA75NUc0J2ldumlLdEQeCWRYlp2Av0YuIiPURcXlE\nXElRK+SxiPijNC6AioGImymqOYErN5n1n7NwG8J3JV1CcR/UHuCLqd2Vm8z6TTfuVIyIJ0gFWiNi\nSYt1slVuciCY5VCRedgdCGaZVCAPHAhmOfjhJjM7JW85+J5xIJhl0f81F8pwIJhl4jEEMzulAong\nQDDLwWMIZlZTkdsQHAhm2VQgERwIZjmkpx0HnQPBLItqFGZwIJhlUoE8cCCYZeGrDGb2IRXoIjgQ\nzDKoymVHT6FmlomGVGopvb2PFmqZK+mZVJDl+5LOS+0z0vvx9PmVDdtYn9pflbSs0z4dCGY5qFwY\nTHGcoVaopebrwMaIuAo4BqxJ7WuAY6l9Y1oPSfMp5mS8hqIew/2Shtvt0IFglknOQJhcqCVNrLoE\nqFVt2kYx0SoUhVq2pdcPATem9VcC2yPivYh4nWLOxdpMzU05EMxy0JQqN51OoZaLgHci4mR631h0\npV6QJX1+PK3vQi1mPVP+bOB0C7V0nQPBLIPMU6jVCrWsAM6nKPb6dxQ1G89JvYDGoiu1giwHJZ0D\nXAi8jQu1mPVOrmKvLQq1/CHwOPC5tNoI8Eh6PZrekz5/LE3NPgqsSlch5lJUh3623b7dQzDLQYLu\n36n4FWC7pK8CLwKbU/tm4DuSxoEJihAhIvZK2gHsA04CayPig3Y7cCCYZTLU/UItr9HkKkFE/Az4\nfIvvbwA2lN1fqVMGScvTjQ3jktaV3bjZtCGKHkKZpY91DIR0I8N9wE3AfGB1uuHBzBpkqvXaU2V6\nCIuA8Yh4LSL+l6I89cruHpbZYKldZch8p+JZVyYQSt3cIOn22o0W73Mi1/GZDY4KdBGyDSpGxCZg\nE8AFuixybddsIJS8pNjvygTClG9uMJuOqjCnYplThueAeenRy/MornGOdvewzAbQdDhliIiTku4A\ndgHDwJaI2Nv1IzMbJAJV4L7fUmMIEbET2NnlYzEbWMWMSf39v38ZvlPRLJc+v6RYhgPBLIdpdJXB\nzEqoQB44EMxy0fDgjyo6EMxy6P8riqU4EMxy8aCimUF1LjsO/kmPWT/IWJdB0vmSnpX0H5L2Svrr\n1L5V0uuS9qRlQWqXpHvTfCUvSbq2YVsjkvanZaTVPmvcQzDLJV8H4T1gSUS8K+lc4IeS/iV99uWI\neGjS+jdRzJc4D1gMPAAsljQbuBNYCATwgqTRiDjWasfuIZhlknGS1YiId9Pbc9PS7gnilcCD6XtP\nU8zOfCmwDBiLiIkUAmMUFZxaciCY5SAYGh4qtZTaXFHXcQ9whOIf9TPpow3ptGCjpBmprdWcJVMu\n1OJAMMtFJZcSlZsi4oOIWEAx3cAiSb8GrAd+FfgNYDbFLMxZeQzBLIMpXmVoW7mpUUS8I+lxYHlE\nfCM1vyfp28BfpPet5iw5BNwwqf2JdvtzD8Esk1zTIUi6RNLM9PpjwO8AP0njArXCrzcDL6evjAK3\npqsN1wPHI+IwxZQFSyXNkjQLWJraWnIPwSyTjLchXApsSzOeDwE7IuJRSY9JuoSiQ7IH+GJafyew\ngqK68wngNoCImJB0D8UkRwB3R8REux07EMxyyPi0Y0S8BPx6k/YlLdYPYG2Lz7YAW8ru24FglkkF\nblR0IJjlILpTyu1scyCYZVKFZxkcCGY5+PFnM2ukjA8z9IoDwSyD4sakXh/FmXMgmGXiQDCzOg8q\nmllB/V/qvQwHglkmFeggOBDMcqjKnIoOBLNMKpAHDgSzXNxDMLOC71Q0sxoBQxVIBAeCWSYVyAMH\nglkuVRhD8JyKZplknFOxVeWmuZKeSRWavi/pvNQ+I70fT59f2bCt9an9VUnLOu3bgWCWQ8kiLSV7\nEbXKTZ8GFgDL0+SpXwc2RsRVwDFgTVp/DXAstW9M6yFpPrAKuIaiQMv9aZ7GlhwIZhnUnnbM0UNo\nU7lpCVAr47aNYuZlKCo3bUuvHwJuTDMzrwS2R8R7EfE6xSSsi9rt24FglskUAqFjoZbJlZuA/wLe\niYiTaZXGKkz1Ck3p8+PARZxG5SYPKpplMoU5FTsWaomID4AFqT7DwxQVm7quK4Fw1YJfYvTx9d3Y\ntCU3zdrQ60OovP88cbj8yurOVYaGyk2/SVHE9ZzUC6hVZ4JTlZsOSjoHuBB4m9YVnVryKYNZLuVr\nO7bfTPPKTa8AjwOfS6uNAI+k16PpPenzx1KthlFgVboKMZeiXPyz7fbtUwazDDI/7diqctM+YLuk\nrwIvApvT+puB70gaByYoriwQEXsl7QD2ASeBtelUpCUHglkmZ6Fy02s0uUoQET8DPt9iWxuA0ueX\nDgSzTCpwo6IDwSyLjLUde8mBYJaBS7mZ2Ye4h2BmBU+QYmaN3EMwMyDdh+AxBDOrqUAHwYFgloUv\nO5pZI192NDPAlZvMbJIK5IEDwSyL2hxqA86BYJaJTxnMrK4CeeBAMMtDlbgxyVOomWUgFZcdyyyd\nt6UrJD0uaV8q1PKl1H6XpEOS9qRlRcN3mhZkkbQ8tY1LWtdp3+4hmGWScQzhJPDnEfEjSRcAL0ga\nS59tjIhvTNpvY0GWy4B/k/Qr6eP7KOZkPAg8J2k0Iva12rEDwSyTXHkQEYeBw+n1TyW9Qvt6CvWC\nLMDraW7F2lRr42nqNSRtT+u2DASfMphlkrGUW+M2r6SYX/GZ1HSHpJckbZE0K7W1Ksgy5UItDgSz\nDKQpBULHyk3FNvXzwD8BfxYR/wM8AHyKot7jYeCbuf8cPmUwy2QK//l3rNwk6VyKMPhuRPwzQES8\n1fD5t4BH09t2BVlcqMXs7BNDQ0Ollo5bKroRm4FXIuJvGtovbVjts8DL6XWrgizPAfNSGfnzKAYe\nR9vt2z0Es0wy3pj0GeCPgR+ngq8AfwWslrSAohL0AeAL0L4gi6Q7gF3AMLAlIva227EDwSwDKd+M\nSRHxQ5oXfdvZ5jtNC7JExM5235vMgWCWiW9dNrM6lank2uccCGa5DH4eOBDMcvHjz2YGFGHgORXN\nrK4CHQQHglkuPmUws7oK5IEDwSwLuYdgZklFJl12IJjl4kAws7qhCiSCA8EskyqMIXR8ODtN1XRE\n0sud1jWbrooZk8ot/azMBClbgeVdPg6zAVdu+rR+70V0PGWIiCfTRI9m1kaf/1svJdsYQpoo8naA\nKy6/osPaZtXT7//7l5FtTsWI2BQRCyNi4UUXX5xrs2aDQyWXTptpXblptqQxSfvTz1mpXZLuTdWZ\nXpJ0bcO2RtL6+yWNdNq3J1k1yyBnKTdOVW6aD1wPrE3VmdYBuyNiHrA7vQe4iWJi1XkUvfQHimPS\nbOBOYDFF4ZY7G2o5NOVAMMsk16BiRByOiB+l1z8FapWbVgLb0mrbgJvT65XAg1F4GpiZZmheBoxF\nxEREHAPG6HCBoMxlx+8BTwFXSzooaU3HP5HZNDSFM4ZShVrgI5Wb5qQybwBvAnPS62yVm8pcZVjd\naR0zm9KgYsdCLWl7H6rc1Lj9iAhJcVoH2oZPGcwyyXljUrPKTcBbtWIt6eeR1N6qclO7ik5NORDM\nMig7flCmF9GqchNF1aXalYIR4JGG9lvT1YbrgePp1GIXsFTSrDSYuDS1teRnGcwyOQuVm74G7Ejj\neG8At6TPdgIrgHHgBHAbQERMSLqHoqQbwN0RMdFuxw4Es0xyTbLapnITwI1N1g9gbYttbQG2lN23\nA8EskwrcqOhAMMtBFZlCzYOKZlbnHoJZJlXoITgQzDKpQB44EMxycSCYWZ3LwZvZKYOfBw4EsxwG\nYQLVMhwIZlnIpwxm1mDw88CBYJZLBfLAgWCWizI93NRLDgSzTAY/DhwIZlkU5eAHPxL8cJOZ1TkQ\nzHLIXOy1WZFlSXdJOiRpT1pWNHy2PhVqeVXSsob25altXNK6yfuZzIFglknmYq9baV5DYWNELEjL\nzrTf+cAq4Jr0nfslDUsaBu6jKOQyH1id1m3JYwhmfWiKRZZXAtsj4j3gdUnjFJWaAMYj4jUASdvT\nuvtabcg9BLMsxJDKLWfojlS/cUtDWbZshVocCGa5lC/dVLpy0yQPAJ8CFgCHgW9mPX58ymCWRXHZ\nsfTqpSo3TRYRb9X3J30LeDS9bVeQxYVazHohUzX41ttPVZuSzwK1KxCjwCpJMyTNpagC/SxFPYZ5\nkuZKOo9i4HG03T7cQzDLYYpdhI6bK4os30BxenGQoqz7DZIWAAEcAL4AEBF7Je2gGCw8CayNiA/S\ndu6gqNY0DGyJiL3t9utAMMsk532KLYosb26z/gZgQ5P2nRSVnUpxIJhlUoE7lx0IZrlU4VkGB4JZ\nBpmHEHrGgWCWzeAnggPBLJMq9BB8H4KZ1bmHYJaDp2E3sw8b/ERwIJhl4h6CmZ3iQGhuz54Xj144\n6+NvdGPbXXIxcLTXB1Fxg/g7/mTZFeXKTa1FxCXd2G63SHr+dB5HtfL8Ox4MPmUwy6QKYwi+D8HM\n6txDKGzq9QFMA9X+HcsPN1VGRFT7L2sfmBa/48HPAweCWS4VyAOPIZhlk7F0U4vKTbMljUnan37O\nSu2SdG+qzvSSpGsbvjOS1t8vaaTTfh0IZv1pKx+t3LQO2B0R84Dd6T0UlZnmpeV2iunakTSbYi7G\nxRSFW+5sqOXQlAPBLJOcsy5HxJPAxKTmlcC29HobcHND+4NReBqYmWZoXgaMRcRERBwDxmheHq7O\nYwhmuZQfRLhY0vMN7zeVHHSdExGH0+s3gTnpdbbKTQ4EswyKKdRKJ8JpFWppFBEhKc5kG834lMEs\nk24XagHeqhVrST+PpPZWlZvaVXRqyoFglkPZNDizRBgFalcKRoBHGtpvTVcbrgeOp1OLXcBSSbPS\nYOLS1NaSTxnMsul65aavATskrQHeAG5Jq+8EVgDjwAngNoCImJB0D0VJN4C7I2LyQOWH9xuR/TTE\nbNq57rrr4ql/f6bUujM+du4L/frkp3sIZrlU4FZFB4JZJhXIAweCWR7VmHbZgWCWSQXywJcdzewU\n9xDMMpjinYp9y5cdzTKQ9AOKmaXLOBoRbR8y6hUHgpnVeQzBzOocCGZW50AwszoHgpnVORDMrM6B\nYGZ1DgQzq3MgmFmdA8HM6v4fIyyJK2F1gksAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13992b978>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# SVM accuracy\n",
    "print(\"SVM Accuracy:\\n{}\\n\".format(svm_evaluation[0]))\n",
    "print(\"SVM F1 Score:\",svm_evaluation[2])\n",
    "\n",
    "print(\"SVM Confusion Matrix:\\n{}\\n\".format(svm_evaluation[1]))\n",
    "plt.matshow(svm_evaluation[1], cmap='Purples')\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SVM gives over 83% accuracy with f1 score of 0.834"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Table for model performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1 Score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Logistic Regression</th>\n",
       "      <td>0.925816</td>\n",
       "      <td>0.926048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Naive Bayes'</th>\n",
       "      <td>0.869266</td>\n",
       "      <td>0.862397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SVM</th>\n",
       "      <td>0.837225</td>\n",
       "      <td>0.834707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Neural Network</th>\n",
       "      <td>0.930535</td>\n",
       "      <td>0.931527</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Accuracy  F1 Score\n",
       "Model                                  \n",
       "Logistic Regression  0.925816  0.926048\n",
       "Naive Bayes'         0.869266  0.862397\n",
       "SVM                  0.837225  0.834707\n",
       "Neural Network       0.930535  0.931527"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracies = [logistic_evaluation[0], nb_evaluation[0], svm_evaluation[0], nn_evaluation[0]]\n",
    "f1_scores = [logistic_evaluation[2], nb_evaluation[2], svm_evaluation[2], nn_evaluation[2]]\n",
    "model_names = ['Logistic Regression', \"Naive Bayes'\", 'SVM', 'Neural Network']\n",
    "\n",
    "pd.DataFrame(\n",
    "    {'Model': model_names,\n",
    "     'Accuracy': accuracies,\n",
    "     'F1 Score': f1_scores\n",
    "    }).set_index('Model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:python-got3]",
   "language": "python",
   "name": "conda-env-python-got3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}