movie-reviews-sentiment-analysis-scikit-learn.ipynb

{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.9 (tags/v3.10.9:1dd9be6, Dec  6 2022, 20:01:21) [MSC v.1934 64 bit (AMD64)]"},"vscode":{"interpreter":{"hash":"2407659e7e7968990f76a627cce7446618739ede0dc4bbd0e9edff98f921e070"}},"colab":{"provenance":[],"collapsed_sections":["RRc7rBv02kd7","Rrlqk0NJezo4","MudajbXmefzm","h2EATGhw-1bF","EPGn4gmf-4ne","SO7NTEHJIb9g"]},"gpuClass":"standard"},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Sources**\n* IMDB Dataset: https://www.kaggle.com/datasets/lakshmi25npathi/imdb-dataset-of-50k-movie-reviews","metadata":{"id":"_jaY_2UC2kd3"}},{"cell_type":"markdown","source":"# **Data**","metadata":{"id":"RRc7rBv02kd7"}},{"cell_type":"code","source":"from google.colab import drive\ndrive.mount('/content/drive')\n\nurl = \"/content/drive/MyDrive/Temp/Datasets/IMDB Dataset.csv\"","metadata":{"id":"H0AuPcLH3GC7","colab":{"base_uri":"https://localhost:8080/"},"outputId":"bffbb3a9-35bf-492c-8333-0bde96d11e4f"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":"Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"}]},{"cell_type":"code","source":"import pandas as pd\ndf = pd.read_csv(url)\n\ndf.drop_duplicates(inplace=True)\n\ndf.head()","metadata":{"id":"aXdiN53-2kd8","colab":{"base_uri":"https://localhost:8080/","height":206},"outputId":"828221c3-e35c-480b-fb42-ff9ee39ca222"},"execution_count":null,"outputs":[{"output_type":"execute_result","execution_count":2,"data":{"text/plain":["                                              review sentiment\n","0  One of the other reviewers has mentioned that ...  positive\n","1  A wonderful little production. <br /><br />The...  positive\n","2  I thought this was a wonderful way to spend ti...  positive\n","3  Basically there's a family where a little boy ...  negative\n","4  Petter Mattei's \"Love in the Time of Money\" is...  positive"],"text/html":["\n","  <div id=\"df-5c3ba73d-e296-4cef-9ef0-bed1e628f799\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>review</th>\n","      <th>sentiment</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>One of the other reviewers has mentioned that ...</td>\n","      <td>positive</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>A wonderful little production. &lt;br /&gt;&lt;br /&gt;The...</td>\n","      <td>positive</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>I thought this was a wonderful way to spend ti...</td>\n","      <td>positive</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Basically there's a family where a little boy ...</td>\n","      <td>negative</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Petter Mattei's \"Love in the Time of Money\" is...</td>\n","      <td>positive</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5c3ba73d-e296-4cef-9ef0-bed1e628f799')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-5c3ba73d-e296-4cef-9ef0-bed1e628f799 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-5c3ba73d-e296-4cef-9ef0-bed1e628f799');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{}}]},{"cell_type":"markdown","source":"## Preparing Y","metadata":{"id":"_Qg_yh_LGW_S"}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nle = LabelEncoder()\nY = le.fit_transform(df.sentiment)\nCLASSES = le.classes_","metadata":{"id":"zWb6QOStGQP9"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preparing X","metadata":{"id":"8PUnB9RdGNbG"}},{"cell_type":"markdown","source":"### Preprocessing","metadata":{"id":"5cYn4e3xG-_w"}},{"cell_type":"code","source":"!pip install emoji","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"zDwA6nWNGfyR","outputId":"de3deaf0-677c-4f0b-df09-235fc8e997c4"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":"Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n\nRequirement already satisfied: emoji in /usr/local/lib/python3.8/dist-packages (2.2.0)\n"}]},{"cell_type":"code","source":"import nltk\nnltk.download('stopwords')\nnltk.download('punkt')","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9Si2QmaJGQNn","outputId":"f769e21d-ad9f-497a-94ea-7863922b0047"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":"[nltk_data] Downloading package stopwords to /root/nltk_data...\n\n[nltk_data]   Package stopwords is already up-to-date!\n\n[nltk_data] Downloading package punkt to /root/nltk_data...\n\n[nltk_data]   Package punkt is already up-to-date!\n"},{"output_type":"execute_result","execution_count":5,"data":{"text/plain":["True"]},"metadata":{}}]},{"cell_type":"code","source":"import emoji\nfrom nltk.tokenize import word_tokenize\n\nimport string\npunc = string.punctuation\nabbv = {\n    \"AFAIK\":\"as far as I know\",\n\t\"IMO\":\t\"in my opinion\",\n\t\"IMHO\":\t\"in my humble opinion\",\n\t\"LGTM\":\t\"look good to me\",\n\t\"AKA\":\t\"also know as\",\n\t\"ASAP\":\t\"as sone as possible\",\n\t\"BTW\":\t\"by the way\",\n\t\"FAQ\":\t\"frequently asked questions\",\n\t\"DIY\":\t\"do it yourself\",\n\t\"DM\":\t\"direct message\",\n\t\"FYI\":\t\"for your information\",\n\t\"IC\":\t\"i see\",\n\t\"IOW\":\t\"in other words\",\n\t\"IIRC\":\t\"If I Remember Correctly\",\n\t\"icymi\":\"In case you missed it\",\n\t\"CUZ\":\t\"because\",\n\t\"COS\":\t\"because\",\n\t\"nv\":\t\"nevermind\",\n\t\"PLZ\":\t\"please\",\n}\n\nfrom nltk.corpus import stopwords\nstopwords.words('english')\n\nimport re\nhtml_pattern = re.compile('<.*?>')\nurls_pattern = re.compile(r'https?://\\S+|www\\.\\S+')\nemoji_pattern = re.compile(\"[\"\n\tu\"\\U0001F600-\\U0001F64F\"  # emoticons\n\tu\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n\tu\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n\tu\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n\"]+\", flags=re.UNICODE)\n\n\nfrom nltk.stem.porter import PorterStemmer\nps = PorterStemmer()\n\ndef preprocess(text):\n\n    # Lowercase\n    text = text.lower()\n\n    # HTML Tags\n    text = html_pattern.sub(r'', text)\n\n    # urls\n    text = urls_pattern.sub(r'', text)\n\n    # punctuations\n    text = text.translate(str.maketrans(\"\", \"\", punc))\n\n    # Emojis\n    text = emoji.demojize(text)\n    text = emoji_pattern.sub(r'', text)\n\n    new_text = []\n\n    for word in text.split(\" \"):\n\n        # abbreviations\n        word = abbv.get(word.upper(), word)\n            \n        # Stemming\n        word = ps.stem(word)\n\n        new_text.append(word)\n\n    text = \" \".join(new_text)\n\n    return text\n\npreprocess(\"This is the best movie I have ever watched\")","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"id":"jfxm6pxuGc8g","outputId":"0295bdf8-ee36-4ae0-9f7d-ad3a305d7153"},"execution_count":null,"outputs":[{"output_type":"execute_result","execution_count":6,"data":{"text/plain":["'thi is the best movi i have ever watch'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{}}]},{"cell_type":"code","source":"# from tqdm import tqdm\n\n# # cleaned = df.review.apply(preprocess)\n\n# cleaned = []\n# for i in tqdm(df.review):\n#     cleaned.append(preprocess(i))","metadata":{"id":"XUXjFn2qHj5I"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\n\n# WRITTING\n# with open(\"/content/drive/MyDrive/Temp/dumps/cleaned_reviews1.json\", 'w') as f:\n#     json.dump(cleaned, f)\n\n# READING\nwith open(\"/content/drive/MyDrive/Temp/dumps/cleaned_reviews1.json\", 'rb') as f:\n    cleaned = json.load(f)","metadata":{"id":"4TsWJsc_Ila5"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Splitting","metadata":{"id":"rX2zk4SfHM1e"}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ncleaned_train, cleaned_test, Y_train, Y_test = train_test_split(\n\tcleaned,\n\tY,\n\ttest_size=0.2,\n\trandom_state=42,\n\tstratify=Y\n)","metadata":{"id":"pSgpHX8DJFJ1"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Hyper Paramters**","metadata":{"id":"Rrlqk0NJezo4"}},{"cell_type":"code","source":"MAX_FEATURES = 5000\nNGRAM_RANGE = (1, 1)","metadata":{"id":"mh4m5E_Gey1p"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Encoders**","metadata":{"id":"MudajbXmefzm"}},{"cell_type":"code","source":"encoders = {}\n\n# TfidfVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nencoders['TfidfVectorizer'] = TfidfVectorizer(\n    lowercase = True,\n    stop_words = 'english',\n    max_features = MAX_FEATURES,\n    binary = False,\n    sublinear_tf = True,\n    ngram_range=NGRAM_RANGE,\n)\n\n# BiGram_tfidf\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nencoders['BiGram_tfidf'] = TfidfVectorizer(\n    lowercase = True,\n    stop_words = 'english',\n    max_features = MAX_FEATURES,\n    binary = False,\n    sublinear_tf = True,\n    ngram_range=(2, 2),\n)\n\n# UniBiGram_tfidf\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nencoders['UniBiGram_tfidf'] = TfidfVectorizer(\n    lowercase = True,\n    stop_words = 'english',\n    max_features = MAX_FEATURES,\n    binary = False,\n    sublinear_tf = True,\n    ngram_range=(1, 2),\n)\n\n# CountVectorizer\nfrom sklearn.feature_extraction.text import CountVectorizer\nencoders['CountVectorizer'] = CountVectorizer(\n    lowercase=True,\n    stop_words='english',\n    max_features=MAX_FEATURES,\n    binary=False,\n    ngram_range=NGRAM_RANGE,\n)\n\n# BiGram\nfrom sklearn.feature_extraction.text import CountVectorizer\nencoders['BiGram'] = CountVectorizer(\n    lowercase=True,\n    stop_words='english',\n    max_features=MAX_FEATURES,\n    binary=False,\n    ngram_range=(2, 2),\n)\n\n# UniBiGram\nfrom sklearn.feature_extraction.text import CountVectorizer\nencoders['UniBiGram'] = CountVectorizer(\n    lowercase=True,\n    stop_words='english',\n    max_features=MAX_FEATURES,\n    binary=False,\n    ngram_range=(1, 2),\n)","metadata":{"id":"U3h676CPeh_1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TAKES TOO MUCH TIME\n\"\"\"\n\n    import gensim\n    from gensim.utils import simple_preprocess\n    from nltk import sent_tokenize\n    from nltk.corpus import stopwords\n    from tqdm import tqdm\n\n    sw_list = stopwords.words('english')\n\n    story = []\n    for doc in tqdm(cleaned_train):\n        raw_sent = sent_tokenize(doc)\n        for sent in raw_sent:\n            sent = \" \".join([i for i in sent.split() if i not in sw_list])\n            story.append(simple_preprocess(sent))\n\n    model = gensim.models.Word2Vec(\n        window=10,\n        min_count=2,\n        size=MAX_FEATURES,\n    )\n    model.build_vocab(story)\n    model.train(story, total_examples=model.corpus_count, epochs=model.epochs)\n\n    import numpy as np\n    def document_vector(doc):\n        return np.mean(\n            [model.wv[i] for i in doc.split() if i in model.wv.index2word],\n            axis=0\n        )\n\n    X_train = []\n    for doc in tqdm(cleaned_train):\n        X_train.append(document_vector(doc))\n\n    X_test = []\n    for doc in tqdm(cleaned_test):\n        X_test.append(document_vector(doc))\n\n\"\"\"\n\"\"","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"id":"vYmMD6DO17pr","outputId":"18272afa-2309-4623-89a6-21f54cc4c63c"},"execution_count":null,"outputs":[{"output_type":"execute_result","execution_count":12,"data":{"text/plain":["''"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{}}]},{"cell_type":"markdown","source":"# **Models**","metadata":{"id":"P-Xk5DWdeaRf"}},{"cell_type":"code","source":"models = {}\n\n# GaussianNB\nfrom sklearn.naive_bayes import GaussianNB, BernoulliNB, MultinomialNB\nmodels['GaussianNB'] = GaussianNB\n\n# BernoulliNB\nmodels['BernoulliNB'] = BernoulliNB\n\n# MultinomialNB\nmodels['MultinomialNB'] = MultinomialNB\n\n# RandomForestClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nmodels['RandomForestClassifier'] = RandomForestClassifier","metadata":{"id":"oJ0bii-_eZ1D"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Pipeline**","metadata":{"id":"zVyMrN2iYLC2"}},{"cell_type":"code","source":"encoders.keys(), models.keys()","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"VtBeBMOPYVx_","outputId":"bcac22f2-3f65-4077-b446-397a1d216752"},"execution_count":null,"outputs":[{"output_type":"execute_result","execution_count":14,"data":{"text/plain":["(dict_keys(['TfidfVectorizer', 'BiGram_tfidf', 'UniBiGram_tfidf', 'CountVectorizer', 'BiGram', 'UniBiGram']),\n"," dict_keys(['GaussianNB', 'BernoulliNB', 'MultinomialNB', 'RandomForestClassifier']))"]},"metadata":{}}]},{"cell_type":"code","source":"# MODELS TO SKIP\nto_skip = {\n    'BiGram_tfidf': ['RandomForestClassifier'],\n    'UniBiGram_tfidf': ['RandomForestClassifier'],\n    'BiGram': ['RandomForestClassifier'],\n    'UniBiGram': ['RandomForestClassifier'],\n}","metadata":{"id":"uKOzJjamYPMV"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipeline = {}\nfor encoder_name, encoder in encoders.items():\n    pipeline[encoder_name] = {\n        \"encoder\": encoder,\n        \"models\": {}\n    }\n    for model_name, model in models.items():\n        if model_name in to_skip.get(encoder_name, []):\n            continue\n        \n        pipeline[encoder_name]['models'][model_name] = model()","metadata":{"id":"C7oKFe4mYb8M"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipeline","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"QpcjyWVFZDeY","outputId":"e8ef74f6-e576-4e2b-cbd4-fd6bb492b576"},"execution_count":null,"outputs":[{"output_type":"execute_result","execution_count":17,"data":{"text/plain":["{'TfidfVectorizer': {'encoder': TfidfVectorizer(max_features=5000, stop_words='english', sublinear_tf=True),\n","  'models': {'GaussianNB': GaussianNB(),\n","   'BernoulliNB': BernoulliNB(),\n","   'MultinomialNB': MultinomialNB(),\n","   'RandomForestClassifier': RandomForestClassifier()}},\n"," 'BiGram_tfidf': {'encoder': TfidfVectorizer(max_features=5000, ngram_range=(2, 2), stop_words='english',\n","                  sublinear_tf=True),\n","  'models': {'GaussianNB': GaussianNB(),\n","   'BernoulliNB': BernoulliNB(),\n","   'MultinomialNB': MultinomialNB()}},\n"," 'UniBiGram_tfidf': {'encoder': TfidfVectorizer(max_features=5000, ngram_range=(1, 2), stop_words='english',\n","                  sublinear_tf=True),\n","  'models': {'GaussianNB': GaussianNB(),\n","   'BernoulliNB': BernoulliNB(),\n","   'MultinomialNB': MultinomialNB()}},\n"," 'CountVectorizer': {'encoder': CountVectorizer(max_features=5000, stop_words='english'),\n","  'models': {'GaussianNB': GaussianNB(),\n","   'BernoulliNB': BernoulliNB(),\n","   'MultinomialNB': MultinomialNB(),\n","   'RandomForestClassifier': RandomForestClassifier()}},\n"," 'BiGram': {'encoder': CountVectorizer(max_features=5000, ngram_range=(2, 2), stop_words='english'),\n","  'models': {'GaussianNB': GaussianNB(),\n","   'BernoulliNB': BernoulliNB(),\n","   'MultinomialNB': MultinomialNB()}},\n"," 'UniBiGram': {'encoder': CountVectorizer(max_features=5000, ngram_range=(1, 2), stop_words='english'),\n","  'models': {'GaussianNB': GaussianNB(),\n","   'BernoulliNB': BernoulliNB(),\n","   'MultinomialNB': MultinomialNB()}}}"]},"metadata":{}}]},{"cell_type":"markdown","source":"# **Training On Data**","metadata":{"id":"GhFpLWus2Yz_"}},{"cell_type":"code","source":"import joblib\nimport time\nsave_path = \"./drive/MyDrive/Temp/dumps/\"\n\npipeline","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"NkAZF-ubeZ3a","outputId":"b5d90f7a-d0d6-409c-e466-b93e451f23e8"},"execution_count":null,"outputs":[{"output_type":"execute_result","execution_count":18,"data":{"text/plain":["{'TfidfVectorizer': {'encoder': TfidfVectorizer(max_features=5000, stop_words='english', sublinear_tf=True),\n","  'models': {'GaussianNB': GaussianNB(),\n","   'BernoulliNB': BernoulliNB(),\n","   'MultinomialNB': MultinomialNB(),\n","   'RandomForestClassifier': RandomForestClassifier()}},\n"," 'BiGram_tfidf': {'encoder': TfidfVectorizer(max_features=5000, ngram_range=(2, 2), stop_words='english',\n","                  sublinear_tf=True),\n","  'models': {'GaussianNB': GaussianNB(),\n","   'BernoulliNB': BernoulliNB(),\n","   'MultinomialNB': MultinomialNB()}},\n"," 'UniBiGram_tfidf': {'encoder': TfidfVectorizer(max_features=5000, ngram_range=(1, 2), stop_words='english',\n","                  sublinear_tf=True),\n","  'models': {'GaussianNB': GaussianNB(),\n","   'BernoulliNB': BernoulliNB(),\n","   'MultinomialNB': MultinomialNB()}},\n"," 'CountVectorizer': {'encoder': CountVectorizer(max_features=5000, stop_words='english'),\n","  'models': {'GaussianNB': GaussianNB(),\n","   'BernoulliNB': BernoulliNB(),\n","   'MultinomialNB': MultinomialNB(),\n","   'RandomForestClassifier': RandomForestClassifier()}},\n"," 'BiGram': {'encoder': CountVectorizer(max_features=5000, ngram_range=(2, 2), stop_words='english'),\n","  'models': {'GaussianNB': GaussianNB(),\n","   'BernoulliNB': BernoulliNB(),\n","   'MultinomialNB': MultinomialNB()}},\n"," 'UniBiGram': {'encoder': CountVectorizer(max_features=5000, ngram_range=(1, 2), stop_words='english'),\n","  'models': {'GaussianNB': GaussianNB(),\n","   'BernoulliNB': BernoulliNB(),\n","   'MultinomialNB': MultinomialNB()}}}"]},"metadata":{}}]},{"cell_type":"markdown","source":"## Fitting on Ecoders","metadata":{"id":"rpJHa3H-Wq7u"}},{"cell_type":"code","source":"for encoder_name, encoder_data in pipeline.items():\n    print(f\"Fitting on Encoder: '{encoder_name}' ......... \", end=\"\")\n    i = time.time()\n\n    encoder_data['encoder'].fit(cleaned_train)\n    joblib.dump(encoder_data['encoder'], f\"{save_path}encoders/{encoder_name}.pkl\")\n\n    print(f\"Done ({round(time.time() - i, 3)}ms)\")","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"VI8SqMWdZg-h","outputId":"583ff591-029e-4dab-81a9-2801809af08b"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":"Fitting on Encoder: 'TfidfVectorizer' ......... Done (7.979ms)\n\nFitting on Encoder: 'BiGram_tfidf' ......... Done (38.287ms)\n\nFitting on Encoder: 'UniBiGram_tfidf' ......... Done (44.096ms)\n\nFitting on Encoder: 'CountVectorizer' ......... Done (7.675ms)\n\nFitting on Encoder: 'BiGram' ......... Done (37.17ms)\n\nFitting on Encoder: 'UniBiGram' ......... Done (43.285ms)\n"}]},{"cell_type":"markdown","source":"## Fitting on Models","metadata":{"id":"qrMNUHUeWyTR"}},{"cell_type":"code","source":"# Fitting the Models\nfor encoder_name, encoder_data in pipeline.items():\n    print(encoder_name)\n    X = encoder_data['encoder'].transform(cleaned_train).toarray()\n    for model_name, model in encoder_data['models'].items():\n        print(f\"\\t Fitting on '{model_name}' ......... \", end=\"\")\n        i = time.time()\n\n        model.fit(X, Y_train)\n        joblib.dump(model, f\"{save_path}encoders/{encoder_name}_{model_name}.pkl\")\n\n        print(f\"Done ({round(time.time() - i, 3)}ms)\")\n","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4zRAmdAZW06J","outputId":"584e8d68-b0df-41b1-a1a4-294d38d96eb7"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":"TfidfVectorizer\n\n\t Fitting on 'GaussianNB' ......... Done (3.624ms)\n\n\t Fitting on 'BernoulliNB' ......... Done (2.371ms)\n\n\t Fitting on 'MultinomialNB' ......... Done (0.866ms)\n\n\t Fitting on 'RandomForestClassifier' ......... Done (151.17ms)\n\nBiGram_tfidf\n\n\t Fitting on 'GaussianNB' ......... Done (3.193ms)\n\n\t Fitting on 'BernoulliNB' ......... Done (2.72ms)\n\n\t Fitting on 'MultinomialNB' ......... Done (0.878ms)\n\nUniBiGram_tfidf\n\n\t Fitting on 'GaussianNB' ......... Done (2.567ms)\n\n\t Fitting on 'BernoulliNB' ......... Done (2.273ms)\n\n\t Fitting on 'MultinomialNB' ......... Done (0.84ms)\n\nCountVectorizer\n\n\t Fitting on 'GaussianNB' ......... Done (3.609ms)\n\n\t Fitting on 'BernoulliNB' ......... Done (12.908ms)\n\n\t Fitting on 'MultinomialNB' ......... Done (6.98ms)\n\n\t Fitting on 'RandomForestClassifier' ......... Done (122.544ms)\n\nBiGram\n\n\t Fitting on 'GaussianNB' ......... Done (3.565ms)\n\n\t Fitting on 'BernoulliNB' ......... Done (7.952ms)\n\n\t Fitting on 'MultinomialNB' ......... Done (8.754ms)\n\nUniBiGram\n\n\t Fitting on 'GaussianNB' ......... Done (3.535ms)\n\n\t Fitting on 'BernoulliNB' ......... Done (13.764ms)\n\n\t Fitting on 'MultinomialNB' ......... Done (6.084ms)\n"}]},{"cell_type":"markdown","source":"# **Testing The Data**","metadata":{"id":"3huGmxNj5Q3x"}},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n\nmetrics = {}\nfor encoder_name, encoder_data in pipeline.items():\n    print(encoder_name)\n    encoder = encoder_data['encoder']\n    X_train = encoder.transform(cleaned_train).toarray()\n    X_test = encoder.transform(cleaned_test).toarray()\n    for model_name, model in encoder_data['models'].items():\n        print(f\"\\t Predicting on '{model_name}' ......... \", end=\"\")\n\n        pred_train = model.predict(X_train)\n        pred_test = model.predict(X_test)\n        metrics[f\"{encoder_name} - {model_name}\"] = {\n            \"train_accuracy\": accuracy_score(Y_train, pred_train),\n            \"train_confusion_matrix\": confusion_matrix(Y_train, pred_train),\n            \"train_classification_report\": classification_report(Y_train, pred_train, zero_division=0),\n            \"test_accuracy\": accuracy_score(Y_test, pred_test),\n            \"test_confusion_matrix\": confusion_matrix(Y_test, pred_test),\n            \"test_classification_report\": classification_report(Y_test, pred_test, zero_division=0)\n        }\n\n        print(\"Done\")","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"qL3eNHcteZ8O","outputId":"ec7b7c32-f787-4900-d3c3-53a0361d2a15"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":"TfidfVectorizer\n\n\t Predicting on 'GaussianNB' ......... Done\n\n\t Predicting on 'BernoulliNB' ......... Done\n\n\t Predicting on 'MultinomialNB' ......... Done\n\n\t Predicting on 'RandomForestClassifier' ......... Done\n\nBiGram_tfidf\n\n\t Predicting on 'GaussianNB' ......... Done\n\n\t Predicting on 'BernoulliNB' ......... Done\n\n\t Predicting on 'MultinomialNB' ......... Done\n\nUniBiGram_tfidf\n\n\t Predicting on 'GaussianNB' ......... Done\n\n\t Predicting on 'BernoulliNB' ......... Done\n\n\t Predicting on 'MultinomialNB' ......... Done\n\nCountVectorizer\n\n\t Predicting on 'GaussianNB' ......... Done\n\n\t Predicting on 'BernoulliNB' ......... Done\n\n\t Predicting on 'MultinomialNB' ......... Done\n\n\t Predicting on 'RandomForestClassifier' ......... Done\n\nBiGram\n\n\t Predicting on 'GaussianNB' ......... Done\n\n\t Predicting on 'BernoulliNB' ......... Done\n\n\t Predicting on 'MultinomialNB' ......... Done\n\nUniBiGram\n\n\t Predicting on 'GaussianNB' ......... Done\n\n\t Predicting on 'BernoulliNB' ......... Done\n\n\t Predicting on 'MultinomialNB' ......... Done\n"}]},{"cell_type":"markdown","source":"# **Preview**","metadata":{"id":"VXBgno4y-yAk"}},{"cell_type":"markdown","source":"## Accuracy","metadata":{"id":"h2EATGhw-1bF"}},{"cell_type":"code","source":"best_train = 0\nbest_train_model = None\nbest_test = 0\nbest_test_model = None\nfor key, val in metrics.items():\n    train_accuracy = val['train_accuracy']\n    test_accuracy = val['test_accuracy']\n\n    if train_accuracy > best_train:\n        best_train = train_accuracy\n        best_train_model = key\n    if test_accuracy > best_test:\n        best_test = test_accuracy\n        best_test_model = key\n\nprint(\"Best Train Model :\", best_train_model, best_train)\nprint(\"Best Test Model  :\", best_test_model, best_test)","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bIXUALI6dbCj","outputId":"797bdd2b-cfb3-4212-d4c1-e3042fe8da27"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":"Best Train Model : TfidfVectorizer - RandomForestClassifier 1.0\n\nBest Test Model  : UniBiGram_tfidf - MultinomialNB 0.8573157204799838\n"}]},{"cell_type":"code","source":"for key, val in metrics.items():\n    print(key)\n    print(\"\\t Train Accuracy: \", val['train_accuracy'])\n    print(\"\\t Test Accuracy:  \", val['test_accuracy'])","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"U9E18Yuw5TjH","outputId":"b1d67e8f-a128-4068-c2ae-d6bc9ccbd487"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":"TfidfVectorizer - GaussianNB\n\n\t Train Accuracy:  0.8204462372368587\n\n\t Test Accuracy:   0.7990319653120903\n\nTfidfVectorizer - BernoulliNB\n\n\t Train Accuracy:  0.8509517206605319\n\n\t Test Accuracy:   0.845921145507714\n\nTfidfVectorizer - MultinomialNB\n\n\t Train Accuracy:  0.8591201310979453\n\n\t Test Accuracy:   0.8544922859735807\n\nTfidfVectorizer - RandomForestClassifier\n\n\t Train Accuracy:  1.0\n\n\t Test Accuracy:   0.8425935262680246\n\nBiGram_tfidf - GaussianNB\n\n\t Train Accuracy:  0.8315139291566872\n\n\t Test Accuracy:   0.8002420086719774\n\nBiGram_tfidf - BernoulliNB\n\n\t Train Accuracy:  0.8226143955628388\n\n\t Test Accuracy:   0.8085106382978723\n\nBiGram_tfidf - MultinomialNB\n\n\t Train Accuracy:  0.8423295096432624\n\n\t Test Accuracy:   0.8249470606030049\n\nUniBiGram_tfidf - GaussianNB\n\n\t Train Accuracy:  0.8504979200806757\n\n\t Test Accuracy:   0.8353332661087022\n\nUniBiGram_tfidf - BernoulliNB\n\n\t Train Accuracy:  0.8567502836253624\n\n\t Test Accuracy:   0.8483412322274881\n\nUniBiGram_tfidf - MultinomialNB\n\n\t Train Accuracy:  0.8634816588932308\n\n\t Test Accuracy:   0.8573157204799838\n\nCountVectorizer - GaussianNB\n\n\t Train Accuracy:  0.7469809655867894\n\n\t Test Accuracy:   0.7310678632651003\n\nCountVectorizer - BernoulliNB\n\n\t Train Accuracy:  0.8509517206605319\n\n\t Test Accuracy:   0.845921145507714\n\nCountVectorizer - MultinomialNB\n\n\t Train Accuracy:  0.8439934451027354\n\n\t Test Accuracy:   0.8402742764949077\n\nCountVectorizer - RandomForestClassifier\n\n\t Train Accuracy:  1.0\n\n\t Test Accuracy:   0.8420893415347384\n\nBiGram - GaussianNB\n\n\t Train Accuracy:  0.8178494894743477\n\n\t Test Accuracy:   0.7918725420994253\n\nBiGram - BernoulliNB\n\n\t Train Accuracy:  0.8226143955628388\n\n\t Test Accuracy:   0.8085106382978723\n\nBiGram - MultinomialNB\n\n\t Train Accuracy:  0.8350939115088869\n\n\t Test Accuracy:   0.8195018654835131\n\nUniBiGram - GaussianNB\n\n\t Train Accuracy:  0.8261187444850624\n\n\t Test Accuracy:   0.8142583442573359\n\nUniBiGram - BernoulliNB\n\n\t Train Accuracy:  0.8567502836253624\n\n\t Test Accuracy:   0.8483412322274881\n\nUniBiGram - MultinomialNB\n\n\t Train Accuracy:  0.8465901928652464\n\n\t Test Accuracy:   0.8422910154280528\n"}]},{"cell_type":"markdown","source":"## Confusion Matrix","metadata":{"id":"EPGn4gmf-4ne"}},{"cell_type":"code","source":"for key, val in metrics.items():\n    print(\" ------ \" + key + \" ------ \")\n    print(\"Train:\\n\", val['train_confusion_matrix'])\n    print(\"Test: \\n\", val['test_confusion_matrix'])","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"V-z7k-u26IxC","outputId":"4282b70c-4e9a-4c18-c8be-f7b5891acb64"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":" ------ TfidfVectorizer - GaussianNB ------ \n\nTrain:\n\n [[16463  3295]\n\n [ 3827 16080]]\n\nTest: \n\n [[4025  915]\n\n [1078 3899]]\n\n ------ TfidfVectorizer - BernoulliNB ------ \n\nTrain:\n\n [[16954  2804]\n\n [ 3108 16799]]\n\nTest: \n\n [[4202  738]\n\n [ 790 4187]]\n\n ------ TfidfVectorizer - MultinomialNB ------ \n\nTrain:\n\n [[16838  2920]\n\n [ 2668 17239]]\n\nTest: \n\n [[4197  743]\n\n [ 700 4277]]\n\n ------ TfidfVectorizer - RandomForestClassifier ------ \n\nTrain:\n\n [[19758     0]\n\n [    0 19907]]\n\nTest: \n\n [[4148  792]\n\n [ 769 4208]]\n\n ------ BiGram_tfidf - GaussianNB ------ \n\nTrain:\n\n [[15999  3759]\n\n [ 2924 16983]]\n\nTest: \n\n [[3844 1096]\n\n [ 885 4092]]\n\n ------ BiGram_tfidf - BernoulliNB ------ \n\nTrain:\n\n [[14923  4835]\n\n [ 2201 17706]]\n\nTest: \n\n [[3635 1305]\n\n [ 594 4383]]\n\n ------ BiGram_tfidf - MultinomialNB ------ \n\nTrain:\n\n [[16202  3556]\n\n [ 2698 17209]]\n\nTest: \n\n [[3964  976]\n\n [ 760 4217]]\n\n ------ UniBiGram_tfidf - GaussianNB ------ \n\nTrain:\n\n [[16622  3136]\n\n [ 2794 17113]]\n\nTest: \n\n [[4102  838]\n\n [ 795 4182]]\n\n ------ UniBiGram_tfidf - BernoulliNB ------ \n\nTrain:\n\n [[16597  3161]\n\n [ 2521 17386]]\n\nTest: \n\n [[4083  857]\n\n [ 647 4330]]\n\n ------ UniBiGram_tfidf - MultinomialNB ------ \n\nTrain:\n\n [[16724  3034]\n\n [ 2381 17526]]\n\nTest: \n\n [[4157  783]\n\n [ 632 4345]]\n\n ------ CountVectorizer - GaussianNB ------ \n\nTrain:\n\n [[17406  2352]\n\n [ 7684 12223]]\n\nTest: \n\n [[4277  663]\n\n [2004 2973]]\n\n ------ CountVectorizer - BernoulliNB ------ \n\nTrain:\n\n [[16954  2804]\n\n [ 3108 16799]]\n\nTest: \n\n [[4202  738]\n\n [ 790 4187]]\n\n ------ CountVectorizer - MultinomialNB ------ \n\nTrain:\n\n [[16904  2854]\n\n [ 3334 16573]]\n\nTest: \n\n [[4208  732]\n\n [ 852 4125]]\n\n ------ CountVectorizer - RandomForestClassifier ------ \n\nTrain:\n\n [[19758     0]\n\n [    0 19907]]\n\nTest: \n\n [[4138  802]\n\n [ 764 4213]]\n\n ------ BiGram - GaussianNB ------ \n\nTrain:\n\n [[15007  4751]\n\n [ 2474 17433]]\n\nTest: \n\n [[3611 1329]\n\n [ 735 4242]]\n\n ------ BiGram - BernoulliNB ------ \n\nTrain:\n\n [[14923  4835]\n\n [ 2201 17706]]\n\nTest: \n\n [[3635 1305]\n\n [ 594 4383]]\n\n ------ BiGram - MultinomialNB ------ \n\nTrain:\n\n [[16026  3732]\n\n [ 2809 17098]]\n\nTest: \n\n [[3932 1008]\n\n [ 782 4195]]\n\n ------ UniBiGram - GaussianNB ------ \n\nTrain:\n\n [[17159  2599]\n\n [ 4298 15609]]\n\nTest: \n\n [[4245  695]\n\n [1147 3830]]\n\n ------ UniBiGram - BernoulliNB ------ \n\nTrain:\n\n [[16597  3161]\n\n [ 2521 17386]]\n\nTest: \n\n [[4083  857]\n\n [ 647 4330]]\n\n ------ UniBiGram - MultinomialNB ------ \n\nTrain:\n\n [[16688  3070]\n\n [ 3015 16892]]\n\nTest: \n\n [[4154  786]\n\n [ 778 4199]]\n"}]},{"cell_type":"markdown","source":"## Classification Report","metadata":{"id":"QbcyHd5e_yQo"}},{"cell_type":"code","source":"for key, val in metrics.items():\n    print(\" ------ \" + key + \" ------ \")\n    print(\" -- Train -- \")\n    print(val['train_classification_report'])\n    print(\" -- Test -- \")\n    print(val['test_classification_report'])","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ek1Num7b6IzM","outputId":"1fb2fbc0-52e2-45f9-c9a1-63d3d31c6f46"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":" ------ TfidfVectorizer - GaussianNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.81      0.83      0.82     19758\n\n           1       0.83      0.81      0.82     19907\n\n\n\n    accuracy                           0.82     39665\n\n   macro avg       0.82      0.82      0.82     39665\n\nweighted avg       0.82      0.82      0.82     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.79      0.81      0.80      4940\n\n           1       0.81      0.78      0.80      4977\n\n\n\n    accuracy                           0.80      9917\n\n   macro avg       0.80      0.80      0.80      9917\n\nweighted avg       0.80      0.80      0.80      9917\n\n\n\n ------ TfidfVectorizer - BernoulliNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.85      0.86      0.85     19758\n\n           1       0.86      0.84      0.85     19907\n\n\n\n    accuracy                           0.85     39665\n\n   macro avg       0.85      0.85      0.85     39665\n\nweighted avg       0.85      0.85      0.85     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.84      0.85      0.85      4940\n\n           1       0.85      0.84      0.85      4977\n\n\n\n    accuracy                           0.85      9917\n\n   macro avg       0.85      0.85      0.85      9917\n\nweighted avg       0.85      0.85      0.85      9917\n\n\n\n ------ TfidfVectorizer - MultinomialNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.86      0.85      0.86     19758\n\n           1       0.86      0.87      0.86     19907\n\n\n\n    accuracy                           0.86     39665\n\n   macro avg       0.86      0.86      0.86     39665\n\nweighted avg       0.86      0.86      0.86     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.86      0.85      0.85      4940\n\n           1       0.85      0.86      0.86      4977\n\n\n\n    accuracy                           0.85      9917\n\n   macro avg       0.85      0.85      0.85      9917\n\nweighted avg       0.85      0.85      0.85      9917\n\n\n\n ------ TfidfVectorizer - RandomForestClassifier ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       1.00      1.00      1.00     19758\n\n           1       1.00      1.00      1.00     19907\n\n\n\n    accuracy                           1.00     39665\n\n   macro avg       1.00      1.00      1.00     39665\n\nweighted avg       1.00      1.00      1.00     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.84      0.84      0.84      4940\n\n           1       0.84      0.85      0.84      4977\n\n\n\n    accuracy                           0.84      9917\n\n   macro avg       0.84      0.84      0.84      9917\n\nweighted avg       0.84      0.84      0.84      9917\n\n\n\n ------ BiGram_tfidf - GaussianNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.85      0.81      0.83     19758\n\n           1       0.82      0.85      0.84     19907\n\n\n\n    accuracy                           0.83     39665\n\n   macro avg       0.83      0.83      0.83     39665\n\nweighted avg       0.83      0.83      0.83     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.81      0.78      0.80      4940\n\n           1       0.79      0.82      0.81      4977\n\n\n\n    accuracy                           0.80      9917\n\n   macro avg       0.80      0.80      0.80      9917\n\nweighted avg       0.80      0.80      0.80      9917\n\n\n\n ------ BiGram_tfidf - BernoulliNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.87      0.76      0.81     19758\n\n           1       0.79      0.89      0.83     19907\n\n\n\n    accuracy                           0.82     39665\n\n   macro avg       0.83      0.82      0.82     39665\n\nweighted avg       0.83      0.82      0.82     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.86      0.74      0.79      4940\n\n           1       0.77      0.88      0.82      4977\n\n\n\n    accuracy                           0.81      9917\n\n   macro avg       0.82      0.81      0.81      9917\n\nweighted avg       0.81      0.81      0.81      9917\n\n\n\n ------ BiGram_tfidf - MultinomialNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.86      0.82      0.84     19758\n\n           1       0.83      0.86      0.85     19907\n\n\n\n    accuracy                           0.84     39665\n\n   macro avg       0.84      0.84      0.84     39665\n\nweighted avg       0.84      0.84      0.84     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.84      0.80      0.82      4940\n\n           1       0.81      0.85      0.83      4977\n\n\n\n    accuracy                           0.82      9917\n\n   macro avg       0.83      0.82      0.82      9917\n\nweighted avg       0.83      0.82      0.82      9917\n\n\n\n ------ UniBiGram_tfidf - GaussianNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.86      0.84      0.85     19758\n\n           1       0.85      0.86      0.85     19907\n\n\n\n    accuracy                           0.85     39665\n\n   macro avg       0.85      0.85      0.85     39665\n\nweighted avg       0.85      0.85      0.85     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.84      0.83      0.83      4940\n\n           1       0.83      0.84      0.84      4977\n\n\n\n    accuracy                           0.84      9917\n\n   macro avg       0.84      0.84      0.84      9917\n\nweighted avg       0.84      0.84      0.84      9917\n\n\n\n ------ UniBiGram_tfidf - BernoulliNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.87      0.84      0.85     19758\n\n           1       0.85      0.87      0.86     19907\n\n\n\n    accuracy                           0.86     39665\n\n   macro avg       0.86      0.86      0.86     39665\n\nweighted avg       0.86      0.86      0.86     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.86      0.83      0.84      4940\n\n           1       0.83      0.87      0.85      4977\n\n\n\n    accuracy                           0.85      9917\n\n   macro avg       0.85      0.85      0.85      9917\n\nweighted avg       0.85      0.85      0.85      9917\n\n\n\n ------ UniBiGram_tfidf - MultinomialNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.88      0.85      0.86     19758\n\n           1       0.85      0.88      0.87     19907\n\n\n\n    accuracy                           0.86     39665\n\n   macro avg       0.86      0.86      0.86     39665\n\nweighted avg       0.86      0.86      0.86     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.87      0.84      0.85      4940\n\n           1       0.85      0.87      0.86      4977\n\n\n\n    accuracy                           0.86      9917\n\n   macro avg       0.86      0.86      0.86      9917\n\nweighted avg       0.86      0.86      0.86      9917\n\n\n\n ------ CountVectorizer - GaussianNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.69      0.88      0.78     19758\n\n           1       0.84      0.61      0.71     19907\n\n\n\n    accuracy                           0.75     39665\n\n   macro avg       0.77      0.75      0.74     39665\n\nweighted avg       0.77      0.75      0.74     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.68      0.87      0.76      4940\n\n           1       0.82      0.60      0.69      4977\n\n\n\n    accuracy                           0.73      9917\n\n   macro avg       0.75      0.73      0.73      9917\n\nweighted avg       0.75      0.73      0.73      9917\n\n\n\n ------ CountVectorizer - BernoulliNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.85      0.86      0.85     19758\n\n           1       0.86      0.84      0.85     19907\n\n\n\n    accuracy                           0.85     39665\n\n   macro avg       0.85      0.85      0.85     39665\n\nweighted avg       0.85      0.85      0.85     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.84      0.85      0.85      4940\n\n           1       0.85      0.84      0.85      4977\n\n\n\n    accuracy                           0.85      9917\n\n   macro avg       0.85      0.85      0.85      9917\n\nweighted avg       0.85      0.85      0.85      9917\n\n\n\n ------ CountVectorizer - MultinomialNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.84      0.86      0.85     19758\n\n           1       0.85      0.83      0.84     19907\n\n\n\n    accuracy                           0.84     39665\n\n   macro avg       0.84      0.84      0.84     39665\n\nweighted avg       0.84      0.84      0.84     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.83      0.85      0.84      4940\n\n           1       0.85      0.83      0.84      4977\n\n\n\n    accuracy                           0.84      9917\n\n   macro avg       0.84      0.84      0.84      9917\n\nweighted avg       0.84      0.84      0.84      9917\n\n\n\n ------ CountVectorizer - RandomForestClassifier ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       1.00      1.00      1.00     19758\n\n           1       1.00      1.00      1.00     19907\n\n\n\n    accuracy                           1.00     39665\n\n   macro avg       1.00      1.00      1.00     39665\n\nweighted avg       1.00      1.00      1.00     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.84      0.84      0.84      4940\n\n           1       0.84      0.85      0.84      4977\n\n\n\n    accuracy                           0.84      9917\n\n   macro avg       0.84      0.84      0.84      9917\n\nweighted avg       0.84      0.84      0.84      9917\n\n\n\n ------ BiGram - GaussianNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.86      0.76      0.81     19758\n\n           1       0.79      0.88      0.83     19907\n\n\n\n    accuracy                           0.82     39665\n\n   macro avg       0.82      0.82      0.82     39665\n\nweighted avg       0.82      0.82      0.82     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.83      0.73      0.78      4940\n\n           1       0.76      0.85      0.80      4977\n\n\n\n    accuracy                           0.79      9917\n\n   macro avg       0.80      0.79      0.79      9917\n\nweighted avg       0.80      0.79      0.79      9917\n\n\n\n ------ BiGram - BernoulliNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.87      0.76      0.81     19758\n\n           1       0.79      0.89      0.83     19907\n\n\n\n    accuracy                           0.82     39665\n\n   macro avg       0.83      0.82      0.82     39665\n\nweighted avg       0.83      0.82      0.82     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.86      0.74      0.79      4940\n\n           1       0.77      0.88      0.82      4977\n\n\n\n    accuracy                           0.81      9917\n\n   macro avg       0.82      0.81      0.81      9917\n\nweighted avg       0.81      0.81      0.81      9917\n\n\n\n ------ BiGram - MultinomialNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.85      0.81      0.83     19758\n\n           1       0.82      0.86      0.84     19907\n\n\n\n    accuracy                           0.84     39665\n\n   macro avg       0.84      0.84      0.83     39665\n\nweighted avg       0.84      0.84      0.83     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.83      0.80      0.81      4940\n\n           1       0.81      0.84      0.82      4977\n\n\n\n    accuracy                           0.82      9917\n\n   macro avg       0.82      0.82      0.82      9917\n\nweighted avg       0.82      0.82      0.82      9917\n\n\n\n ------ UniBiGram - GaussianNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.80      0.87      0.83     19758\n\n           1       0.86      0.78      0.82     19907\n\n\n\n    accuracy                           0.83     39665\n\n   macro avg       0.83      0.83      0.83     39665\n\nweighted avg       0.83      0.83      0.83     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.79      0.86      0.82      4940\n\n           1       0.85      0.77      0.81      4977\n\n\n\n    accuracy                           0.81      9917\n\n   macro avg       0.82      0.81      0.81      9917\n\nweighted avg       0.82      0.81      0.81      9917\n\n\n\n ------ UniBiGram - BernoulliNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.87      0.84      0.85     19758\n\n           1       0.85      0.87      0.86     19907\n\n\n\n    accuracy                           0.86     39665\n\n   macro avg       0.86      0.86      0.86     39665\n\nweighted avg       0.86      0.86      0.86     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.86      0.83      0.84      4940\n\n           1       0.83      0.87      0.85      4977\n\n\n\n    accuracy                           0.85      9917\n\n   macro avg       0.85      0.85      0.85      9917\n\nweighted avg       0.85      0.85      0.85      9917\n\n\n\n ------ UniBiGram - MultinomialNB ------ \n\n -- Train -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.85      0.84      0.85     19758\n\n           1       0.85      0.85      0.85     19907\n\n\n\n    accuracy                           0.85     39665\n\n   macro avg       0.85      0.85      0.85     39665\n\nweighted avg       0.85      0.85      0.85     39665\n\n\n\n -- Test -- \n\n              precision    recall  f1-score   support\n\n\n\n           0       0.84      0.84      0.84      4940\n\n           1       0.84      0.84      0.84      4977\n\n\n\n    accuracy                           0.84      9917\n\n   macro avg       0.84      0.84      0.84      9917\n\nweighted avg       0.84      0.84      0.84      9917\n\n\n"}]},{"cell_type":"markdown","source":"## Realtime Testing","metadata":{"id":"IYVoO0kCellv"}},{"cell_type":"code","source":"def realtime_test(text, orig, show=True, correct_only=False):\n    cleaned_text = preprocess(text)\n    correct = []\n    for encoder_name, encoder_data in pipeline.items():\n        X = encoder_data['encoder'].transform([cleaned_text]).toarray()\n        for model_name, model in encoder_data['models'].items():\n            pred = model.predict(X)[0]\n            model_label = f\"{encoder_name} - {model_name}\"\n            if (pred == orig):\n                correct.append(model_label)\n                if show:\n                    print(f\"{CLASSES[pred]} - {model_label}\")\n            elif not correct_only and show:\n                print(f\"{CLASSES[pred]} - {model_label}\")\n    if not show:\n        return correct\n\ndef test_many(texts, origs):\n    corrects = []\n    for text, orig in zip(texts, origs):\n        correct = realtime_test(text, orig, show=False)\n        corrects.append(correct)\n    common = set(corrects[0])\n    for correct in corrects:\n        common = set(correct) & common\n    \n    print(\"All sentences correctly predicted by these models:\")\n    for i in common:\n        print(\".\", i, round(metrics[i]['test_accuracy'], 2))","metadata":{"id":"_5MZr2soepZa"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text = \"Greatest Movie of all time\"\norig = 1\n\nrealtime_test(text, orig)","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"c30STuSzepcJ","outputId":"9d35d0b0-ef21-42de-d68f-8a0267a67f2e"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":"positive - TfidfVectorizer - GaussianNB\n\npositive - TfidfVectorizer - BernoulliNB\n\npositive - TfidfVectorizer - MultinomialNB\n\npositive - TfidfVectorizer - RandomForestClassifier\n\npositive - BiGram_tfidf - GaussianNB\n\npositive - BiGram_tfidf - BernoulliNB\n\npositive - BiGram_tfidf - MultinomialNB\n\npositive - UniBiGram_tfidf - GaussianNB\n\npositive - UniBiGram_tfidf - BernoulliNB\n\npositive - UniBiGram_tfidf - MultinomialNB\n\nnegative - CountVectorizer - GaussianNB\n\npositive - CountVectorizer - BernoulliNB\n\npositive - CountVectorizer - MultinomialNB\n\npositive - CountVectorizer - RandomForestClassifier\n\npositive - BiGram - GaussianNB\n\npositive - BiGram - BernoulliNB\n\npositive - BiGram - MultinomialNB\n\nnegative - UniBiGram - GaussianNB\n\npositive - UniBiGram - BernoulliNB\n\npositive - UniBiGram - MultinomialNB\n"}]},{"cell_type":"code","source":"text = \"Greatest Movie of all time\"\norig = 1\n\nrealtime_test(text, orig, correct_only=True)","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"56p1LiAHepex","outputId":"1ae825c4-73f3-434c-e812-3322b61ece91"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":"positive - TfidfVectorizer - GaussianNB\n\npositive - TfidfVectorizer - BernoulliNB\n\npositive - TfidfVectorizer - MultinomialNB\n\npositive - TfidfVectorizer - RandomForestClassifier\n\npositive - BiGram_tfidf - GaussianNB\n\npositive - BiGram_tfidf - BernoulliNB\n\npositive - BiGram_tfidf - MultinomialNB\n\npositive - UniBiGram_tfidf - GaussianNB\n\npositive - UniBiGram_tfidf - BernoulliNB\n\npositive - UniBiGram_tfidf - MultinomialNB\n\npositive - CountVectorizer - BernoulliNB\n\npositive - CountVectorizer - MultinomialNB\n\npositive - CountVectorizer - RandomForestClassifier\n\npositive - BiGram - GaussianNB\n\npositive - BiGram - BernoulliNB\n\npositive - BiGram - MultinomialNB\n\npositive - UniBiGram - BernoulliNB\n\npositive - UniBiGram - MultinomialNB\n"}]},{"cell_type":"code","source":"text = \"Worst Movie, really hate it\"\norig = 0\n\nrealtime_test(text, orig, correct_only=True)","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"n8jcQVEkephf","outputId":"c965e543-944d-4307-c3d6-d27683f072bb"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":"negative - TfidfVectorizer - GaussianNB\n\nnegative - TfidfVectorizer - BernoulliNB\n\nnegative - TfidfVectorizer - MultinomialNB\n\nnegative - TfidfVectorizer - RandomForestClassifier\n\nnegative - BiGram_tfidf - GaussianNB\n\nnegative - BiGram_tfidf - BernoulliNB\n\nnegative - BiGram_tfidf - MultinomialNB\n\nnegative - UniBiGram_tfidf - GaussianNB\n\nnegative - UniBiGram_tfidf - BernoulliNB\n\nnegative - UniBiGram_tfidf - MultinomialNB\n\nnegative - CountVectorizer - GaussianNB\n\nnegative - CountVectorizer - BernoulliNB\n\nnegative - CountVectorizer - MultinomialNB\n\nnegative - CountVectorizer - RandomForestClassifier\n\nnegative - BiGram - GaussianNB\n\nnegative - BiGram - BernoulliNB\n\nnegative - BiGram - MultinomialNB\n\nnegative - UniBiGram - GaussianNB\n\nnegative - UniBiGram - BernoulliNB\n\nnegative - UniBiGram - MultinomialNB\n"}]},{"cell_type":"code","source":"test_many(\n    [\n        \"amazing movie, really love it\",\n        \"this is going to be my favourite movie of all the time\",\n        \"this movie sucks, I completely hate it\",\n        \"boring movie\",\n    ],\n    [1, 1, 0, 0]\n)","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Sh8HhHRYeplE","outputId":"f2988259-bb72-42bc-f803-0ad85293b986"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":"All sentences correctly predicted by these models:\n\n. CountVectorizer - MultinomialNB 0.84\n\n. CountVectorizer - RandomForestClassifier 0.84\n\n. BiGram_tfidf - BernoulliNB 0.81\n\n. BiGram_tfidf - GaussianNB 0.8\n\n. UniBiGram_tfidf - GaussianNB 0.84\n\n. BiGram - GaussianNB 0.79\n\n. UniBiGram - MultinomialNB 0.84\n\n. CountVectorizer - BernoulliNB 0.85\n\n. UniBiGram_tfidf - MultinomialNB 0.86\n\n. BiGram - MultinomialNB 0.82\n\n. BiGram_tfidf - MultinomialNB 0.82\n\n. TfidfVectorizer - GaussianNB 0.8\n\n. BiGram - BernoulliNB 0.81\n\n. TfidfVectorizer - RandomForestClassifier 0.84\n\n. TfidfVectorizer - MultinomialNB 0.85\n\n. TfidfVectorizer - BernoulliNB 0.85\n"}]},{"cell_type":"code","source":"test_many(\n    [\n        \"amazing movie, really love it\",\n        \"this is going to be my favourite movie of all the time\",\n        \"this movie sucks, I completely hate it\",\n        \"boring movie\",\n        'not bad',\n    ],\n    [1, 1, 0, 0, 1]\n)","metadata":{"id":"Ujz8qgbDIaZw","colab":{"base_uri":"https://localhost:8080/"},"outputId":"6b3d61ca-9716-4fb9-bc90-a2f0407c487f"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":"All sentences correctly predicted by these models:\n\n. BiGram - GaussianNB 0.79\n\n. BiGram_tfidf - BernoulliNB 0.81\n\n. BiGram_tfidf - GaussianNB 0.8\n\n. BiGram - MultinomialNB 0.82\n\n. BiGram_tfidf - MultinomialNB 0.82\n\n. BiGram - BernoulliNB 0.81\n"}]},{"cell_type":"code","source":"","metadata":{"id":"A6D_9DZgIapC"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"_djfBkmTIart"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"4DQmRY1OIauj"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"jICkYtYeIawm"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"f9SQ53FTIayp"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"8UDqh3IhIa0t"},"execution_count":null,"outputs":[]}]}