Train_sentiment_model_final.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4LvN6dm7dhnN"
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from time import time\n",
    "import pickle\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfTransformer, TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, cross_val_score\n",
    "from sklearn.pipeline import make_pipeline, Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import make_scorer, accuracy_score, f1_score\n",
    "from sklearn.metrics import confusion_matrix, roc_auc_score,recall_score, precision_score, classification_report\n",
    "\n",
    "\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training 4 different models for sentiment analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zm3x-jqQeAPE"
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"./datasets/clean_sent_160k_train.csv\",low_memory=False,error_bad_lines=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Drop all rows with NaN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 134
    },
    "colab_type": "code",
    "id": "6HsiXFevefUD",
    "outputId": "9913a516-4f9c-4f9a-945d-809de4df629c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1592046 entries, 0 to 1592045\n",
      "Data columns (total 2 columns):\n",
      "sentiment    1592046 non-null int64\n",
      "text         1592046 non-null object\n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 24.3+ MB\n"
     ]
    }
   ],
   "source": [
    "df.dropna(inplace=True)\n",
    "df.reset_index(drop=True,inplace=True)\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0APWcPMFefEg"
   },
   "source": [
    "### Split into testing and training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GQhTvPmEee1s"
   },
   "outputs": [],
   "source": [
    "train, test = train_test_split(df, test_size=0.2, random_state=1)\n",
    "x_train = train['text'].values\n",
    "x_test = test['text'].values\n",
    "y_train = train['sentiment']\n",
    "y_test = test['sentiment']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "2Art-fNSemTV",
    "outputId": "fa69e6fc-f87a-4dca-f640-e86d54444a0f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/aveek/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "nltk.download('stopwords')\n",
    "stop_words = stopwords.words('english')\n",
    "added = ['.',',','-',';',':','--','\\\"','(',')', '\\'s','?','n\\'t', '<', '>',\n",
    "         '``', '\\'\\'', 'I', 'i', 'a', 'A', '..', '...', 'i\\'m', 'I\\'m']\n",
    "stop_words.extend(added)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HuztlXG89wg7"
   },
   "source": [
    "## Note:\n",
    " Training the models or fitting the vectorizer may take a long time, so they have been pickled and stored as \"*.sav\" files which can be loaded using pickle again without training. The vectorizer was about 250MB, so it has been compressed and stored as \".xz\" "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use tfidf with trigrams to vectorize the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "T07uX9zumn3s"
   },
   "outputs": [],
   "source": [
    "# vectorizer = TfidfVectorizer(stop_words=None , max_features=100000, ngram_range=(1,3))\n",
    "# train_vectors = vectorizer.fit_transform(train['text'])\n",
    "# test_vectors = vectorizer.transform(test['text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save or load vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from joblib import dump, load\n",
    "\n",
    "\n",
    "# Uncomment the next line to save the created vectorizer\n",
    "# filename = 'models/tfidf.xz'\n",
    "# dump(vectorizer, filename)\n",
    "\n",
    "# uncomment the next line to load the saved vectorizer\n",
    "vectorizer = load('./models/tfidf.xz')\n",
    "train_vectors = vectorizer.fit_transform(train['text'])\n",
    "test_vectors = vectorizer.transform(test['text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training the models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "z8h8FQ6Rqxhd"
   },
   "source": [
    "## 1. Linear SVC with L1-based feature selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 84
    },
    "colab_type": "code",
    "id": "sVK4MJppmnr_",
    "outputId": "3f6c4dcd-3ebb-4f1d-ef16-585c568d1de4"
   },
   "outputs": [],
   "source": [
    "Linear_SVC = LinearSVC(penalty=\"l1\", dual=False)\n",
    "Linear_SVC.fit(train_vectors, train['sentiment'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 184
    },
    "colab_type": "code",
    "id": "gDkVbBMYmng8",
    "outputId": "6e8ac0fb-a767-4737-cfc0-1c84c01f8a06"
   },
   "outputs": [],
   "source": [
    "prediction_linear = Linear_SVC.predict(test_vectors)\n",
    "\n",
    "\n",
    "report = classification_report(test['sentiment'], prediction_linear)\n",
    "acc_svc = accuracy_score(test['sentiment'], prediction_linear)\n",
    "\n",
    "print(report)\n",
    "print(\"accuracy:\",acc_svc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save the model for future use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3olmPg6m15I7"
   },
   "outputs": [],
   "source": [
    "# filename = './models/linear_svc.sav'\n",
    "# pickle.dump(Linear_SVC, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "T5WLIbH4q_E2"
   },
   "source": [
    "## 2. Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 137
    },
    "colab_type": "code",
    "id": "wj-TH5A9n3Ut",
    "outputId": "8522e3c7-c234-4ddd-a563-39d59f6b420d"
   },
   "outputs": [],
   "source": [
    "Linear_regression = LogisticRegression()\n",
    "Linear_regression.fit(train_vectors, train['sentiment'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 184
    },
    "colab_type": "code",
    "id": "XidRmhiGn3RJ",
    "outputId": "eccef14a-2c91-48a1-9d7f-0a6f8d668e52"
   },
   "outputs": [],
   "source": [
    "prediction_linear = Linear_regression.predict(test_vectors)\n",
    "\n",
    "\n",
    "report = classification_report(test['sentiment'], prediction_linear)\n",
    "acc_reg = accuracy_score(test['sentiment'], prediction_linear)\n",
    "print(report)\n",
    "print(\"accuracy:\", acc_reg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CK3smYXN2-3C"
   },
   "outputs": [],
   "source": [
    "# filename = './models/linear_regression.sav'\n",
    "# pickle.dump(Linear_regression, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sP116sxEtOmE"
   },
   "source": [
    "## 3. Multinomial Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "RjS813bntOMH",
    "outputId": "bfddfb0d-b685-401d-e785-569475af0d94"
   },
   "outputs": [],
   "source": [
    "classifier_NB = MultinomialNB()\n",
    "classifier_NB.fit(train_vectors, train['sentiment'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 184
    },
    "colab_type": "code",
    "id": "gnK_xo7tn3PG",
    "outputId": "e693c90f-e14b-4f54-95f3-c870dbad991e"
   },
   "outputs": [],
   "source": [
    "prediction_linear = classifier_NB.predict(test_vectors)\n",
    "\n",
    "\n",
    "report = classification_report(test['sentiment'], prediction_linear)\n",
    "acc_nb = accuracy_score(test['sentiment'], prediction_linear)\n",
    "print(report)\n",
    "print(\"accuracy:\",acc_nb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pgFKWjHu3FAj"
   },
   "outputs": [],
   "source": [
    "# filename = './models/multinomial_NB.sav'\n",
    "# pickle.dump(classifier_NB, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dMHP2nI-uSq4"
   },
   "source": [
    "## 4. LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "f3Ro886MvEa_"
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Embedding, LSTM, SpatialDropout1D, Dropout\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FPmvr68u3PRn"
   },
   "source": [
    "### Reducing the number of rows, to make training time reasonable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 134
    },
    "colab_type": "code",
    "id": "r4oNJsi905oy",
    "outputId": "e1726172-021a-4e9d-c99c-d5eacf183cb2"
   },
   "outputs": [],
   "source": [
    "small_df = df[::20]\n",
    "small_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 288
    },
    "colab_type": "code",
    "id": "F7hx1YmcPNun",
    "outputId": "a43ea625-c560-4a22-b0c1-748f281f94e4"
   },
   "outputs": [],
   "source": [
    "import re\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "nltk.download('wordnet')\n",
    "wordnet_lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "extra_clean = []\n",
    "\n",
    "for line in small_df['text']:\n",
    "  tweet = re.sub(\"[^a-zA-Z]\", \" \", line)\n",
    "  tweet = [wordnet_lemmatizer.lemmatize(x) for x in tweet.split(\" \") if\n",
    "             x not in stop_words and len(x) > 2]\n",
    "  \n",
    "  extra_clean.append(\" \".join(tweet))\n",
    "  \n",
    "small_df['text'] = extra_clean\n",
    "small_df = small_df.drop(small_df[small_df['text'] == ''].index)\n",
    "small_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get vocabulary size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "E-bzdrfpuGQ2",
    "outputId": "d0ab4232-e87a-4a26-9120-e395a8803aa0"
   },
   "outputs": [],
   "source": [
    "vocab = []\n",
    "for x in small_df['text']:\n",
    "    for word in x.split(' '):\n",
    "        vocab.append(word)\n",
    "print(len(set(vocab)), len(vocab))\n",
    "\n",
    "vocab_size = len(set(vocab)) +500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "M1PPHTGm15Jk",
    "outputId": "f9094569-a398-4dfb-b5a6-4b2fd33a5c14"
   },
   "outputs": [],
   "source": [
    "print('Maximum review length: {}'.format(\n",
    "len(max((small_df['text']), key=len))))\n",
    "\n",
    "print('Minimum review length: {}'.format(\n",
    "len(min((small_df['text']), key=len))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenize and pad the input sequences for the LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "46ySJfwjuGC_"
   },
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words=20000)\n",
    "\n",
    "max_words = 20\n",
    "\n",
    "tokenizer.fit_on_texts(small_df['text'].values)\n",
    "X = tokenizer.texts_to_sequences(small_df['text'].values)\n",
    "X = pad_sequences(X, maxlen=max_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create the LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NhgbdGJWwuxM"
   },
   "outputs": [],
   "source": [
    "\n",
    "embed_dim = 32\n",
    "lstm_out = 32\n",
    "batch_size= 128\n",
    "\n",
    "# #Buidling the LSTM network\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(20000, embed_dim, input_length = max_words))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(LSTM(lstm_out))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(1,activation='sigmoid'))\n",
    "model.compile(loss = 'binary_crossentropy', optimizer='adam',metrics = ['accuracy'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IsKx033LuFzu"
   },
   "outputs": [],
   "source": [
    "Y = small_df['sentiment']\n",
    "X_train, X_valid, Y_train, Y_valid = train_test_split(X,Y, test_size = 0.20, random_state = 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Nhr-yJdf15Ju"
   },
   "source": [
    "### Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 67
    },
    "colab_type": "code",
    "id": "jgogasctwXcw",
    "outputId": "04596a9b-13dc-4fcf-aba0-4b96ccdbc183"
   },
   "outputs": [],
   "source": [
    "\n",
    "model.fit(X_train, Y_train, epochs = 1, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "4wrRQOl-wXZu",
    "outputId": "14d866cc-00b5-4a79-f07e-a1f42fae2d7c"
   },
   "outputs": [],
   "source": [
    "score,acc = model.evaluate(X_valid, Y_valid, batch_size = batch_size, verbose = 0)\n",
    "print(\"Logloss score: %.2f\" % (score))\n",
    "print(\"Accuracy: %.5f\" % acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MSvHDy7dwXXY"
   },
   "outputs": [],
   "source": [
    "# filename = './models/LSTM.sav'\n",
    "# pickle.dump(model, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare the accuracy of the 4 methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 101
    },
    "colab_type": "code",
    "id": "AbLWYPydAJ0Z",
    "outputId": "464da54a-0d95-491d-f56f-cfb4b6092d82"
   },
   "outputs": [],
   "source": [
    "print(\"Accuracy- \")\n",
    "print(\"SVC: %.3f\" % acc_svc)\n",
    "print(\"Linear Reg: %.3f\" % acc_reg)\n",
    "print(\"Naive Bayes: %.3f\" % acc_nb)\n",
    "print(\"LSTM: %.3f\" % acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "BmMIV-LZ4GkZ"
   },
   "source": [
    "# Observation\n",
    "\n",
    "### Of the 4 methods we used logistic regression gives us the best accuracy. With SVM close behind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Train_sentiment_model.ipynb",
   "provenance": [],
   "toc_visible": true,
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}