Text Classification - Shallow Machine Learning.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 210,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "71d305a0",
        "outputId": "74afbcfd-4f32-4bbd-d96c-76de857d928b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (3.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from nltk) (1.1.0)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.7/dist-packages (from nltk) (2022.6.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from nltk) (4.64.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from nltk) (7.1.2)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: Sastrawi in /usr/local/lib/python3.7/dist-packages (1.0.1)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: xgboost in /usr/local/lib/python3.7/dist-packages (0.90)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from xgboost) (1.7.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from xgboost) (1.21.6)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "!pip install nltk\n",
        "!pip install Sastrawi\n",
        "!pip install xgboost\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import warnings\n",
        "\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "import Sastrawi\n",
        "import re\n",
        "from Sastrawi.Stemmer.StemmerFactory import StemmerFactory\n",
        "from nltk.tokenize import RegexpTokenizer\n",
        "\n",
        "# Modeling Utilities\n",
        "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
        "from sklearn.model_selection import cross_validate, GridSearchCV, RepeatedStratifiedKFold\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score, f1_score, classification_report\n",
        "\n",
        "# Models\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from xgboost import XGBClassifier\n",
        "\n",
        "# For resampling\n",
        "from collections import Counter\n",
        "from sklearn.datasets import make_classification\n",
        "from imblearn.over_sampling import SMOTE, SMOTENC, SMOTEN, ADASYN\n",
        "from imblearn.under_sampling import NearMiss\n",
        "from imblearn.combine import SMOTEENN\n",
        "\n",
        "# Ensemble Classifiers\n",
        "from sklearn.ensemble import VotingClassifier, StackingClassifier\n",
        "\n",
        "# Hyperparameter Tuning\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.model_selection import cross_val_score\n",
        "\n",
        "warnings.filterwarnings(\"ignore\")"
      ],
      "id": "71d305a0"
    },
    {
      "cell_type": "code",
      "execution_count": 211,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9hR8laSjtOMa",
        "outputId": "42d80555-fb43-4b52-9f1b-2295ca838ba6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "id": "9hR8laSjtOMa"
    },
    {
      "cell_type": "code",
      "execution_count": 213,
      "metadata": {
        "collapsed": true,
        "id": "04166af0"
      },
      "outputs": [],
      "source": [
        "# Membaca data\n",
        "path = \"/content/drive/Shared drives/MLL + NLP/Text Classification NLP/dataset/\"\n",
        "train = pd.read_csv(path+\"train.csv\",encoding=\"Latin-1\")\n",
        "test = pd.read_csv(path+\"test.csv\",encoding=\"Latin-1\")\n",
        "dev = pd.read_csv(path+\"dev.csv\",encoding=\"Latin-1\")"
      ],
      "id": "04166af0"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_rvros28t2Nu"
      },
      "source": [
        "Overview data"
      ],
      "id": "_rvros28t2Nu"
    },
    {
      "cell_type": "code",
      "execution_count": 214,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "QgpgUkmJuUoA",
        "outputId": "e0634738-f9a3-4d4a-90e6-6b1d922549fc"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Unnamed: 0                                             text_a label\n",
              "0           0  betewe buka twitter cuman ngetweet liat home b...    no\n",
              "1           1  mas piyuuu mugo2 corona tuh mulut tersumpal ma...    no\n",
              "2           2  e100ss gini buka informasi sejelas nya identit...   yes\n",
              "3           3  neng solo wes ono terduga corona cobo neng ati...    no\n",
              "4           4  midiahn nii akun gak takut takut nya isu coron...    no"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-26f990aa-04a9-49c6-9f70-094d82fa8c33\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>text_a</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>betewe buka twitter cuman ngetweet liat home b...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>mas piyuuu mugo2 corona tuh mulut tersumpal ma...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>e100ss gini buka informasi sejelas nya identit...</td>\n",
              "      <td>yes</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>neng solo wes ono terduga corona cobo neng ati...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>midiahn nii akun gak takut takut nya isu coron...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-26f990aa-04a9-49c6-9f70-094d82fa8c33')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-26f990aa-04a9-49c6-9f70-094d82fa8c33 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-26f990aa-04a9-49c6-9f70-094d82fa8c33');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 214
        }
      ],
      "source": [
        "train.head()"
      ],
      "id": "QgpgUkmJuUoA"
    },
    {
      "cell_type": "code",
      "execution_count": 215,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "6PWWOXgK5fqn",
        "outputId": "a58413e0-31b5-4d08-aff9-0ecf5a235fef"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                              text_a label\n",
              "0  betewe buka twitter cuman ngetweet liat home b...    no\n",
              "1  mas piyuuu mugo2 corona tuh mulut tersumpal ma...    no\n",
              "2  e100ss gini buka informasi sejelas nya identit...   yes\n",
              "3  neng solo wes ono terduga corona cobo neng ati...    no\n",
              "4  midiahn nii akun gak takut takut nya isu coron...    no"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6d6eb287-9da3-4cfd-8e06-01b65b56c326\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text_a</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>betewe buka twitter cuman ngetweet liat home b...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>mas piyuuu mugo2 corona tuh mulut tersumpal ma...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>e100ss gini buka informasi sejelas nya identit...</td>\n",
              "      <td>yes</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>neng solo wes ono terduga corona cobo neng ati...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>midiahn nii akun gak takut takut nya isu coron...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6d6eb287-9da3-4cfd-8e06-01b65b56c326')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6d6eb287-9da3-4cfd-8e06-01b65b56c326 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6d6eb287-9da3-4cfd-8e06-01b65b56c326');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 215
        }
      ],
      "source": [
        "train = train.drop(columns=['Unnamed: 0'])\n",
        "train.head()"
      ],
      "id": "6PWWOXgK5fqn"
    },
    {
      "cell_type": "code",
      "execution_count": 216,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TdoQfn-3t33N",
        "outputId": "69dbe053-f41f-4325-93ae-3c25b8dfd30f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train shape\t: (21601, 2)\n",
            "Test shape\t: (2800, 2)\n",
            "Dev shape\t: (2800, 2)\n"
          ]
        }
      ],
      "source": [
        "print(\"Train shape\\t:\",train.shape)\n",
        "print(\"Test shape\\t:\",test.shape)\n",
        "print(\"Dev shape\\t:\",dev.shape)"
      ],
      "id": "TdoQfn-3t33N"
    },
    {
      "cell_type": "code",
      "execution_count": 217,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0ec0f9a8",
        "outputId": "9f9d28fe-cc4d-4804-9964-ead1425d09e7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Yes: 6089 \t 0.2818850979121337\n",
            "No: 15512 \t 0.7181149020878663\n"
          ]
        }
      ],
      "source": [
        "# Jumlah data berlabel positif dan negatif pada data training\n",
        "count_yes = train[train['label']=='yes'].shape[0]\n",
        "count_no = train[train['label']=='no'].shape[0]\n",
        "percentage_yes = count_yes / (count_yes + count_no)\n",
        "percentage_no = count_no / (count_yes + count_no)\n",
        "print(\"Yes:\",count_yes,\"\\t\",percentage_yes)\n",
        "print(\"No:\",count_no,\"\\t\",percentage_no)"
      ],
      "id": "0ec0f9a8"
    },
    {
      "cell_type": "code",
      "execution_count": 218,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "DWIRWPaRQCC1",
        "outputId": "fcc35f91-4dea-4653-a76e-c1d20b22e3cf"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEWCAYAAACaBstRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAUPUlEQVR4nO3de7RcZXnH8e8TEohIyiVBxEQMWBEwQoJJSeQiEZVLLJa2VsG2IYha6XLVS1GCSygRl1FcFuQi3rgol4YCabtAIEIp0FZIA0ZAIIRLoCdcjSYlcjHA0z/2PnBySM55yZl9Zk74ftaalb3fvWfPM++cmV/2fmf2jsxEkqQSw9pdgCRp6DA0JEnFDA1JUjFDQ5JUzNCQJBUzNCRJxQwNqWERcVVEzGx3HVIrhL/TkF4pIlb3mN0ceA54oZ7/VGZeOPhVSe1naEj9iIhlwNGZee06lg3PzOcHvyqpPTw8Jb0KEbF/RHRFxJci4jHg3IjYOiKuiIgnI+K39fS4Hvf5j4g4up4+MiL+MyK+Va/7YEQc3LYnJL1Khob06r0R2AZ4C/BJqvfRufX8DsAzwBl93H8vYAkwBvgm8KOIiCYLllrF0JBevReBEzPzucx8JjNXZOZlmfl0Zj4FfA14Tx/3fygzf5CZLwDnA9sD2w1C3dKADW93AdIQ9GRmPts9ExGbA/8IHARsXTePiohN6mDo7bHuicx8ut7J2KLBeqWWcU9DevV6f3vkC8Dbgb0y8w+A/ep2Dzlpo2NoSAM3imocY2VEbAOc2OZ6pMYYGtLAnQq8Dvg1cDNwdXvLkZrj7zQkScXc05AkFTM0JEnFDA1JUjFDQ5JUbEj+uG/MmDE5fvz4dpchSUPKrbfe+uvM3HYg2xiSoTF+/HgWLVrU7jIkaUiJiIcGug0PT0mSihkakqRihoYkqdiQHNOQpPVZs2YNXV1dPPvss/2vvJEaOXIk48aNY8SIES3ftqEhaaPS1dXFqFGjGD9+PK/Fa1tlJitWrKCrq4sdd9yx5dv38JSkjcqzzz7L6NGjX5OBARARjB49urE9LUND0kbntRoY3Zp8/oaGJKmYYxqSNmrjj7uypdtbNndGn8tXrlzJRRddxDHHHPOqt7148WIeeeQRDjnkkA0tr3HuaUhSC61cuZKzzjprg+67ePFifvrTn7a4otYyNCSphY477jjuv/9+Jk6cyLHHHsspp5zClClT2H333TnxxOpKwPPnz+eAAw4gM3n00UfZeeedefjhhznhhBOYN28eEydOZN68eevc/sKFC5k2bRqTJk3i3e9+N0uWLBnMp+fhKUlqpblz53LnnXeyePFiFixYwKWXXsrChQvJTA499FBuvPFGDjvsMC677DLOPPNMrr76ak466SR22GEH5syZw6JFizjjjDPWu/1ddtmFm266ieHDh3Pttddy/PHHc9lllw3a8zM0JKkhCxYsYMGCBUyaNAmA1atXs3TpUvbbbz9OP/10JkyYwNSpUzn88MOLt7lq1SpmzpzJ0qVLiQjWrFnTVPnrZGhIUkMyk9mzZ/OpT33qFcu6uroYNmwYjz/+OC+++CLDhpWNFnzlK19h+vTpzJ8/n2XLlrH//vu3uOq+OaYhSS00atQonnrqKQAOPPBAzjnnHFavXg3A8uXLeeKJJ3j++ec56qijuPjii9l111359re//Yr7rs+qVasYO3YsAOedd15zT2Q93NOQtFHr7yuyrTZ69Gj23ntvJkyYwMEHH8wRRxzBtGnTANhiiy244IILOPvss9l3333ZZ5992GOPPZgyZQozZsxg+vTpzJ07l4kTJzJ79mw+8pGPvGL7X/ziF5k5cyYnn3wyM2YM7nMDiMwc9AcdqMmTJ6cXYZK0LnfffTe77rpru8tou3X1Q0TcmpmTB7JdD09Jkop5eEqSOtC5557Laaedtlbb3nvvzZlnntmmiiqGhiR1oFmzZjFr1qx2l/EKHp6StNEZimO1rdTk8zc0JG1URo4cyYoVK16zwdF9EaaRI0c2sn0PT0naqIwbN46uri6efPLJdpfSNt2Xe22CoSFpozJixIhGLnOqioenJEnFDA1JUrEheXjqjuWrWn41LnW+wT4dhKRXck9DklTM0JAkFTM0JEnFDA1JUjFDQ5JUzNCQJBUzNCRJxQwNSVIxQ0OSVMzQkCQVMzQkScUMDUlSMUNDklTM0JAkFTM0JEnFDA1JUjFDQ5JUzNCQJBUzNCRJxQwNSVIxQ0OSVMzQkCQVMzQkScUMDUlSMUNDklRseBMbjYg5wG8y89R6/mvAE8CmwF8AmwHzM/PEiHg9cAkwDtgE+GpmzmuiLknSwDS1p3EO8NcAETEM+CjwGPA24I+AicC7ImI/4CDgkczcIzMnAFeva4MR8cmIWBQRi154elVDZUuS+tJIaGTmMmBFREwCPgD8ApjSY/o2YBeqELkDeH9EfCMi9s3MdSZCZn4/Mydn5uRNNt+yibIlSf1o5PBU7YfAkcAbqfY8DgC+npnf671iROwJHAKcHBHXZeacBuuSJG2gJkNjPjAHGAEcATwPfDUiLszM1RExFlhT1/CbzLwgIlYCRzdYkyRpABoLjcz8fURcD6zMzBeABRGxK/DziABYDfwl8IfAKRHxIlWIfLqpmiRJA9NYaNQD4FOBD3e3ZeZpwGm9Vr0fuKapOiRJrdPIQHhE7AbcB1yXmUubeAxJ0uBrZE8jM+8Cdmpi25Kk9vEX4ZKkYoaGJKmYoSFJKmZoSJKKGRqSpGKGhiSpmKEhSSpmaEiSihkakqRihoYkqZihIUkqZmhIkooZGpKkYoaGJKmYoSFJKmZoSJKKGRqSpGKGhiSpmKEhSSrWyDXCm/bOsVuyaO6MdpchSa857mlIkooZGpKkYoaGJKmYoSFJKmZoSJKKGRqSpGKGhiSpmKEhSSpmaEiSihkakqRihoYkqZihIUkqZmhIkooZGpKkYoaGJKmYoSFJKmZoSJKK9Xnlvoj4076WZ+blrS1HktTJ+rvc6x/3sSwBQ0OSXkP6DI3MnDVYhUiSOl/RmEZEbBcRP4qIq+r53SLi482WJknqNKUD4ecB1wBvqufvBT7bREGSpM5VGhpjMvMS4EWAzHweeKGxqiRJHak0NH4XEaOpBr+JiKnAqsaqkiR1pP6+PdXt88C/AW+NiP8CtgX+vLGqJEkdqSg0MvO2iHgP8HYggCWZuabRyiRJHacoNCJiJHAMsA/VIaqbIuLszHy2yeIkSZ2l9PDUj4GngNPr+SOAnwAfbqIoSVJnKg2NCZm5W4/56yPiriYKkiR1rtJvT91Wf2MKgIjYC1jUTEmSpE7V3wkL76AawxgB/HdEPFzPvwW4p/nyJEmdpL/DUx8clCokSUNCfycsfKjnfES8ARjZaEWSpI5VesLCQyNiKfAgcAOwDLiqwbokSR2odCD8q8BU4N7M3BE4ALi5saokSR2pNDTWZOYKYFhEDMvM64HJDdYlSepApb/TWBkRWwA3AhdGxBPA75orS5LUiUr3ND4EPAN8DrgauJ++LwUrSdoIlZ6wsOdexfkN1SJJ6nD9/bjvKepraPReBGRm/kEjVUmSOlJ/v9MYNViFSJI6X+mYhiRJhoYkqZyhIUkqZmhIkooZGpKkYoaGJKmYoSFJKmZoSJKKGRqSpGKGhiSpmKEhSSpmaEiSipVehKmj3LF8FeOPu7LdZUjSoFo2d0a7S3BPQ5JUztCQJBUzNCRJxQwNSVIxQ0OSVMzQkCQVMzQkScUMDUlSMUNDklTM0JAkFTM0JEnFDA1JUjFDQ5JUzNCQJBUzNCRJxQwNSVIxQ0OSVMzQkCQVMzQkScUMDUlSMUNDklTM0JAkFTM0JEnFDA1JUjFDQ5JUbFBDIyLGR8TdEfGDiPhVRCyIiNdFxMSIuDkibo+I+RGx9WDWJUkq0449jbcBZ2bmO4CVwJ8BPwa+lJm7A3cAJ/a+U0R8MiIWRcSiF55eNagFS5Iq7QiNBzNzcT19K/BWYKvMvKFuOx/Yr/edMvP7mTk5MydvsvmWg1SqJKmndoTGcz2mXwC2akMNkqQN0AkD4auA30bEvvX8XwE39LG+JKlNhre7gNpM4OyI2Bx4AJjV5nokSeswqKGRmcuACT3mv9Vj8dTBrEWS9Op1wuEpSdIQYWhIkooZGpKkYoaGJKmYoSFJKmZoSJKKGRqSpGKGhiSpmKEhSSpmaEiSihkakqRihoYkqZihIUkqZmhIkooZGpKkYoaGJKmYoSFJKmZoSJKKGRqSpGKGhiSpmKEhSSpmaEiSig1vdwEb4p1jt2TR3BntLkOSXnPc05AkFTM0JEnFDA1JUjFDQ5JUzNCQJBUzNCRJxQwNSVIxQ0OSVMzQkCQVMzQkScUMDUlSMUNDklTM0JAkFTM0JEnFDA1JUjFDQ5JUzNCQJBUzNCRJxQwNSVIxQ0OSVMzQkCQVMzQkScUMDUlSMUNDklTM0JAkFTM0JEnFIjPbXcOrFhFPAUvaXUeBMcCv211EP6yxNayxNayxNdZX41syc9uBbHj4QO7cRksyc3K7i+hPRCzq9DqtsTWssTWssTWarNHDU5KkYoaGJKnYUA2N77e7gEJDoU5rbA1rbA1rbI3GahySA+GSpPYYqnsakqQ2MDQkScWGXGhExEERsSQi7ouI4wb5sd8cEddHxF0R8auI+Lu6fZuI+FlELK3/3bpuj4j4Tl3r7RGxZ49tzazXXxoRMxuodZOI+EVEXFHP7xgRt9S1zIuITev2zer5++rl43tsY3bdviQiDmxxfVtFxKURcU9E3B0R0zqtHyPic/XrfGdEXBwRI9vdjxFxTkQ8ERF39mhrWb9FxLsi4o76Pt+JiGhRjafUr/XtETE/IrbqsWyd/bO+9/r6XoNW1Nlj2RciIiNiTD3fMX1Zt3+m7s9fRcQ3e7Q335eZOWRuwCbA/cBOwKbAL4HdBvHxtwf2rKdHAfcCuwHfBI6r248DvlFPHwJcBQQwFbilbt8GeKD+d+t6eusW1/p54CLginr+EuCj9fTZwKfr6WOAs+vpjwLz6und6v7dDNix7vdNWljf+cDR9fSmwFad1I/AWOBB4HU9+u/IdvcjsB+wJ3Bnj7aW9RuwsF436vse3KIaPwAMr6e/0aPGdfYPfbzX1/catKLOuv3NwDXAQ8CYDuzL6cC1wGb1/BsGsy9b9iE1GDdgGnBNj/nZwOw21vOvwPupfp2+fd22PdWPDwG+BxzeY/0l9fLDge/1aF9rvRbUNQ64DngvcEX9R/vrHm/al/qxfnNMq6eH1+tF777tuV4L6tuS6gM5erV3TD9Shcb/1h8Gw+t+PLAT+hEY3+tDpCX9Vi+7p0f7WusNpMZeyw4DLqyn19k/rOe93tffcqvqBC4F9gCW8XJodExfUn3Qv28d6w1KXw61w1Pdb+RuXXXboKsPP0wCbgG2y8xH60WPAdvV0+urt+nncSrwReDFen40sDIzn1/H471US718Vb1+kzXuCDwJnBvVIbQfRsTr6aB+zMzlwLeAh4FHqfrlVjqrH7u1qt/G1tNN1gpwFNX/vDekxr7+lgcsIj4ELM/MX/Za1El9uTOwb31Y6YaImLKBNW5QXw610OgIEbEFcBnw2cz8v57Lsorstn2POSI+CDyRmbe2q4YCw6l2ub+bmZOA31EdVnlJB/Tj1sCHqALuTcDrgYPaVU+pdvdbfyLiy8DzwIXtrqW3iNgcOB44od219GM41R7wVOBY4JINGS/ZUEMtNJZTHW/sNq5uGzQRMYIqMC7MzMvr5scjYvt6+fbAE3X7+upt8nnsDRwaEcuAf6I6RHUasFVEdJ9rrOfjvVRLvXxLYEXDNXYBXZl5Sz1/KVWIdFI/vg94MDOfzMw1wOVUfdtJ/ditVf22vJ5upNaIOBL4IPCxOtw2pMYVrP81GKi3Uv0n4Zf1+2cccFtEvHED6myyL7uAy7OykOqIwpgNqHHD+nJDjwW240aVsA9QvbDdAzrvGMTHD+DHwKm92k9h7YHIb9bTM1h78Gxh3b4N1TH9revbg8A2DdS7Py8PhP8zaw94HVNP/y1rD+BeUk+/g7UH1R6gtQPhNwFvr6f/oe7DjulHYC/gV8Dm9eOeD3ymE/qRVx7jblm/8crB20NaVONBwF3Atr3WW2f/0Md7fX2vQSvq7LVsGS+PaXRSX/4NMKee3pnq0FMMVl+29ENqMG5U32K4l+rbAF8e5Mfeh2rX/3ZgcX07hOrY4HXAUqpvNXT/0QRwZl3rHcDkHts6Crivvs1qqN79eTk0dqr/iO+r/1C6v3kxsp6/r16+U4/7f7mufQkb8M2PfmqbCCyq+/Jf6jdcR/UjcBJwD3An8JP6zdjWfgQuphpjWUP1P86Pt7LfgMn1870fOINeX1YYQI33UX24db9vzu6vf1jPe319r0Er6uy1fBkvh0Yn9eWmwAX1tm8D3juYfelpRCRJxYbamIYkqY0MDUlSMUNDklTM0JAkFTM0JEnFDA2pQER8PSKmR8SfRMTsuu28iHgwIhZHxC8j4oB21yk1zdCQyuwF3Ay8B7ixR/uxmTkR+CzVj6OkjZqhIfWhvg7E7cAU4OfA0cB3I6L3+Yl+Tn2yt6iuu3FufS2FX0TE9Lr9yojYvZ7+Rfc2ImJORHxisJ6TNBCGhtSHzDyW6le451EFx+2ZuXtmzum16kFUv2yH6pQimZnvpDol9vkRMZLq1Cn7RsSWVCft27tef1/W3nuROpahIfVvT6rz9ewC3N1r2SkRcS/VBa++UbftQ3WaBzLzHqqL+exMFRr7UYXFlcAW9ZlVd8zMJU0/CakVhve/ivTaFBETqfYwxlFdrGbzqjkWU12wBqoxjUsj4jPAOcC7+tjk/1Cdj+gB4GdUZyb9BNV1OqQhwT0NaT0yc3E9yN19Wd9/Bw7MzImZ+Uyv1c8AhtXXZb4J+BhAROwM7EB1Nb3fU52078NUYyA3AX+Ph6Y0hBgaUh8iYlvgt5n5IrBLZt61rvWyOvPnyVRXTDyLKkDuAOYBR2bmc/WqN1FdJOuZenpc/a80JHiWW0lSMfc0JEnFDA1JUjFDQ5JUzNCQJBUzNCRJxQwNSVIxQ0OSVOz/AVWK4OJOYRj6AAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "train.groupby(['label']).count().plot(kind='barh')\n",
        "plt.title('Train')\n",
        "plt.xlabel('#Row')\n",
        "plt.show()"
      ],
      "id": "DWIRWPaRQCC1"
    },
    {
      "cell_type": "code",
      "execution_count": 219,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "Dhyt5rqb-Elk",
        "outputId": "b53a78ca-33a6-45cc-91b4-f4854f1dd2d9"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEWCAYAAACEz/viAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAATG0lEQVR4nO3df5hVBZ3H8fcXoViD/IGuFaRgaeqigkGhiMXjrr8ozX6q7UqY/VirZ92edCGfNMmeaN31WVOMrc1f+SNajV2fxzKydZVnNzPQSUxFRNEGCQoFodQQvvvHOdNep2GYme45dwber+e5z5x77rnnfu65d+5nzjl3zonMRJK0cxvU6gCSpNazDCRJloEkyTKQJGEZSJKwDCRJWAaSJCwDabsiYlPDZWtEvNBw/cN9mN9/R8TZVWSV+mpwqwNI/V1mDusYjoiVwNmZeWfrEknN55qB1EcRMSgiZkbEiohYFxHfjYg9y9uGRsQN5fj1EfGziNgnIr4MTAGuLNcsrmzts5AKloHUd58B3gO8A3gD8Bwwt7xtOrAb8EZgBPBJ4IXMvABYBHw6M4dl5qdrTy11wTKQ+u6TwAWZ2Z6ZLwFfBN4fEYOBzRQl8ObM3JKZSzLz+RZmlbrlPgOp7/YDFkTE1oZxW4B9gG9TrBV8JyJ2B26gKI7N9ceUts81A6nvfgmcmJm7N1yGZuaqzNycmRdn5iHAUcC7gDPL+3moYPU7loHUd/OAL0fEfgARsXdEnFIOT42IQyNiF+B5is1GHWsQa4D9WxFY2hbLQOq7y4HbgIURsRG4F3h7edvrgFsoiuAR4G6KTUcd93t/RDwXEV+rN7LUtfDkNpIk1wwkSZaBJMkykCRhGUiSGKD/dLbXXnvl6NGjWx1DkgaUJUuW/CYz9+7qtgFZBqNHj2bx4sWtjiFJA0pEPLWt29xMJEmyDCRJloEkiQG6z0CStmXz5s20t7fz4osvtjpKywwdOpRRo0YxZMiQHt/HMpC0Q2lvb2f48OGMHj2aiGh1nNplJuvWraO9vZ0xY8b0+H5uJpK0Q3nxxRcZMWLETlkEABHBiBEjer1mZBlI2uHsrEXQoS/P3zKQJLnPQNKObfTM25s6v5VzpnV7+/r167nppps455xzej3vtrY2nnnmGU466aS+xusz1wwkqYnWr1/PVVdd1af7trW18f3vf7/JiXrGMpCkJpo5cyYrVqxg3LhxnHfeeVx66aVMnDiRww47jIsuugiABQsWcOyxx5KZrF69mgMPPJCnn36aCy+8kPnz5zNu3Djmz5/f5fzvu+8+jjzySMaPH89RRx3FsmXLmpLbzUSS1ERz5szhoYceoq2tjYULF3LLLbdw3333kZmcfPLJ3HPPPZx66qnceuutzJ07lzvuuIOLL76Yfffdl9mzZ7N48WKuvPLKbc7/oIMOYtGiRQwePJg777yTz3/+89x6661/cm7LQJIqsnDhQhYuXMj48eMB2LRpE8uXL+eYY47hiiuuYOzYsUyaNInTTz+9x/PcsGED06dPZ/ny5UQEmzdvbkpWy0CSKpKZzJo1i0984hN/dFt7ezuDBg1izZo1bN26lUGDerbV/gtf+AJTp05lwYIFrFy5kne+851Nyeo+A0lqouHDh7Nx40YAjj/+eK6++mo2bdoEwKpVq1i7di0vv/wyZ511FjfffDMHH3wwl1122R/dd1s2bNjAyJEjAbj22mublts1A0k7tO19FbTZRowYweTJkxk7diwnnngiZ5xxBkceeSQAw4YN44YbbmDevHlMmTKFo48+msMPP5yJEycybdo0pk6dypw5cxg3bhyzZs3iQx/60B/N//zzz2f69OlccsklTJvWvOcWmdm0mdVlwoQJ6cltJHXlkUce4eCDD251jJbrajlExJLMnNDV9G4mkiS5mUiS+qNrrrmGyy+//BXjJk+ezNy5cyt5PMtAkvqhGTNmMGPGjNoez81EknY4A3FfaDP15flbBpJ2KEOHDmXdunU7bSF0nNxm6NChvbqfm4kk7VBGjRpFe3s7v/71r1sdpWU6TnvZG5aBpB3KkCFDenW6RxXcTCRJsgwkSQN0M9HSVRuafvYiVafuwwFI6j3XDCRJloEkyTKQJGEZSJKwDCRJWAaSJCwDSRKWgSQJy0CShGUgScIykCRhGUiSsAwkSVgGkiQsA0kSloEkCctAkoRlIEnCMpAkYRlIkrAMJElYBpIkLANJEpaBJAnLQJIEDK5iphExG3g2M/+lvP5lYC3wKuCDwKuBBZl5UUS8BvguMArYBfhSZs6vIpckqWtVrRlcDZwJEBGDgNOAXwEHAG8DxgFvjYhjgBOAZzLz8MwcC9zR1Qwj4uMRsTgiFm/53YaKYkvSzqmSMsjMlcC6iBgPHAc8AExsGL4fOIiiHJYCfxURX42IKZnZ5Sd9Zn4jMydk5oRddt2titiStNOqZDNR6d+AjwCvo1hTOBb4Smb+a+cJI+II4CTgkoj4cWbOrjCXJKmTKstgATAbGAKcAbwMfCkibszMTRExEthcZng2M2+IiPXA2RVmkiR1obIyyMzfR8RdwPrM3AIsjIiDgZ9EBMAm4K+BNwOXRsRWinL426oySZK6VlkZlDuOJwEf6BiXmZcDl3eadAXww6pySJK2r5IdyBFxCPA48OPMXF7FY0iSmqeSNYPMfBjYv4p5S5Kaz/9AliRZBpIky0CShGUgScIykCRhGUiSsAwkSVgGkiQsA0kSloEkCctAkoRlIEnCMpAkYRlIkrAMJElYBpIkLANJEpaBJAnLQJJERedArtqhI3dj8ZxprY4hSTsM1wwkSZaBJMkykCRhGUiSsAwkSVgGkiQsA0kSloEkCctAkoRlIEnCMpAkYRlIkrAMJElYBpIkLANJEpaBJAnLQJLEds50FhHv7e72zPxec+NIklphe6e9fHc3tyVgGUjSDqDbMsjMGXUFkSS1To/2GUTEPhHxrYj4QXn9kIj4aLXRJEl16ekO5GuBHwJvKK8/BpxbRSBJUv16WgZ7ZeZ3ga0AmfkysKWyVJKkWvW0DH4bESModhoTEZOADZWlkiTVanvfJurwWeA24E0R8T/A3sD7K0slSapVj8ogM++PiHcAbwECWJaZmytNJkmqTY/KICKGAucAR1NsKloUEfMy88Uqw0mS6tHTzUTXAxuBK8rrZwDfBj5QRShJUr16WgZjM/OQhut3RcTDVQSSJNWvp98mur/8BhEAEfF2YHE1kSRJddvegeqWUuwjGAL8b0Q8XV7fD3i0+niSpDpsbzPRu2pJIUlqqe0dqO6pxusR8efA0EoTSZJq19MD1Z0cEcuBJ4G7gZXADyrMJUmqUU93IH8JmAQ8lpljgGOBeytLJUmqVU/LYHNmrgMGRcSgzLwLmFBhLklSjXr6fwbrI2IYcA9wY0SsBX5bXSxJUp16umZwCvAC8PfAHcAKuj8lpiRpAOnpgeoa1wKuqyiLJKlFtvdPZxspz2HQ+SYgM/O1laSSJNVqe/9nMLyuIJKk1unpPgNJ0g7MMpAkWQaSJMtAkoRlIEnCMpAkYRlIkrAMJElYBpIkLANJEpaBJAnLQJJEz09u068sXbWB0TNvb3UMSarVyjnTKpu3awaSJMtAkmQZSJKwDCRJWAaSJCwDSRKWgSQJy0CShGUgScIykCRhGUiSsAwkSVgGkiQsA0kSloEkCctAkoRlIEnCMpAkYRlIkrAMJElYBpIkLANJEpaBJAnLQJKEZSBJouYyiIjREfFIRHwzIn4REQsj4s8iYlxE3BsRD0bEgojYo85ckrSza8WawQHA3Mz8C2A98D7geuAfMvMwYClwUec7RcTHI2JxRCze8rsNtQaWpB1dK8rgycxsK4eXAG8Cds/Mu8tx1wHHdL5TZn4jMydk5oRddt2tpqiStHNoRRm81DC8Bdi9BRkkSQ36ww7kDcBzETGlvP43wN3dTC9JarLBrQ5Qmg7Mi4hdgSeAGS3OI0k7lVrLIDNXAmMbrv9Tw82T6swiSfp//WEzkSSpxSwDSZJlIEmyDCRJWAaSJCwDSRKWgSQJy0CShGUgScIykCRhGUiSsAwkSVgGkiQsA0kSloEkCctAkoRlIEnCMpAkYRlIkrAMJElYBpIkLANJEjC41QH64tCRu7F4zrRWx5CkHYZrBpIky0CSZBlIkrAMJElYBpIkLANJEpaBJAnLQJKEZSBJwjKQJGEZSJKwDCRJWAaSJCwDSRKWgSQJy0CShGUgScIykCRhGUiSsAwkSVgGkiQsA0kSloEkCctAkoRlIEnCMpAkAZGZrc7QaxGxEVjW6hzbsBfwm1aH6IK5esdcvWOu3mlVrv0yc++ubhhcd5ImWZaZE1odoisRsbg/ZjNX75ird8zVO/0xl5uJJEmWgSRp4JbBN1odoBv9NZu5esdcvWOu3ul3uQbkDmRJUnMN1DUDSVITWQaSpIFXBhFxQkQsi4jHI2JmzY/9xoi4KyIejohfRMTfleO/GBGrIqKtvJzUcJ9ZZdZlEXF8hdlWRsTS8vEXl+P2jIgfRcTy8uce5fiIiK+VuR6MiCMqyvSWhmXSFhHPR8S5rVheEXF1RKyNiIcaxvV6+UTE9HL65RExvaJcl0bEo+VjL4iI3cvxoyPihYblNq/hPm8tX//Hy+xRQa5ev27N/n3dRq75DZlWRkRbOb7O5bWtz4aWv8d6LDMHzAXYBVgB7A+8Cvg5cEiNj/964IhyeDjwGHAI8EXgc11Mf0iZ8dXAmDL7LhVlWwns1WncPwIzy+GZwFfL4ZOAHwABTAJ+WtNr9ytgv1YsL+AY4Ajgob4uH2BP4Iny5x7l8B4V5DoOGFwOf7Uh1+jG6TrN574ya5TZT6wgV69etyp+X7vK1en2fwYubMHy2tZnQ8vfYz29DLQ1g7cBj2fmE5n5e+A7wCl1PXhmrs7M+8vhjcAjwMhu7nIK8J3MfCkznwQep3gOdTkFuK4cvg54T8P467NwL7B7RLy+4izHAisy86lupqlseWXmPcCzXTxeb5bP8cCPMvPZzHwO+BFwQrNzZebCzHy5vHovMKq7eZTZXpuZ92bxiXJ9w3NpWq5ubOt1a/rva3e5yr/uPwjc3N08Klpe2/psaPl7rKcGWhmMBH7ZcL2d7j+MKxMRo4HxwE/LUZ8uV/eu7lgVpN68CSyMiCUR8fFy3D6Zuboc/hWwTwtydTiNV/6Stnp5Qe+XTyuW21kUf0F2GBMRD0TE3RExpRw3ssxSR67evG51L68pwJrMXN4wrvbl1emzYSC8x4CBVwb9QkQMA24Fzs3M54GvA28CxgGrKVZV63Z0Zh4BnAh8KiKOabyx/AuoJd8jjohXAScD/16O6g/L6xVauXy2JSIuAF4GbixHrQb2zczxwGeBmyLitTVG6nevWyen88o/OGpfXl18NvxBf3yPNRpoZbAKeGPD9VHluNpExBCKF/vGzPweQGauycwtmbkV+Cb/v2mjtryZuar8uRZYUGZY07H5p/y5tu5cpROB+zNzTZmx5cur1NvlU1u+iPgI8C7gw+WHCOVmmHXl8BKK7fEHlhkaNyVVkqsPr1udy2sw8F5gfkPeWpdXV58N9OP3WGcDrQx+BhwQEWPKvzZPA26r68HLbZLfAh7JzMsaxjdubz8V6Pimw23AaRHx6ogYAxxAseOq2bleExHDO4YpdkA+VD5+x7cRpgP/2ZDrzPIbDZOADQ2rslV4xV9srV5eDXq7fH4IHBcRe5SbSI4rxzVVRJwAnA+cnJm/axi/d0TsUg7vT7F8niizPR8Rk8r36JkNz6WZuXr7utX5+/qXwKOZ+YfNP3Uur219NtBP32NdqmMvdTMvFHvhH6No+QtqfuyjKVbzHgTaystJwLeBpeX424DXN9zngjLrMv7Ebyx0k2t/im9q/Bz4RcdyAUYAPwaWA3cCe5bjA5hb5loKTKhwmb0GWAfs1jCu9uVFUUargc0U22E/2pflQ7EN//HyMqOiXI9TbDfueI/NK6d9X/n6tgH3A+9umM8Eig/nFcCVlEcXaHKuXr9uzf597SpXOf5a4JOdpq1zeW3rs6Hl77GeXjwchSRpwG0mkiRVwDKQJFkGkiTLQJKEZSBJwjKQeiQivhIRUyPiPRExqxx3bUQ8GcURMX8eEce2OqfUV5aB1DNvpzho3DuAexrGn5eZ44BzgXld3VEaCCwDqRtRnFvgQWAi8BPgbODrEXFhp0l/QnlAsYgYGhHXRHG8/AciYmo5/vaIOKwcfqBjHhExOyI+VtdzkrpiGUjdyMzzKP779lqKQngwMw/LzNmdJj0B+I9y+FPFXfNQikNxXBcRQ4FFwJSI2I3iAHSTy+mn8Mq1Dal2loG0fUdQHOrjIIrj1De6NCIeA26iOBENFIcmuAEgMx8FnqI4QNoiipOzTAZuB4ZFxK7AmMxcVvWTkLozuNUBpP4qIsZRrBGMAn4D7FqMjjbgyHKy8zLzloj4DHA18NZuZvkzimPiPEFx0pK9gI8BSyp5AlIvuGYgbUNmtpU7hztOYfhfwPGZOS4zX+g0+ZXAoCjO/7sI+DBARBwI7Assy+JsX78EPkCxj2ER8DncRKR+wDKQuhERewPPZXEM/4My8+GupsviiI+XUBx6+iqKYlhKcXz9j2TmS+Wki4C1ZZksoljrWFTx05C2y6OWSpJcM5AkWQaSJCwDSRKWgSQJy0CShGUgScIykCQB/wfDZ0kTjN23xAAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "test.groupby(['label']).count().plot(kind='barh')\n",
        "plt.title('Test')\n",
        "plt.xlabel('#Row')\n",
        "plt.show()"
      ],
      "id": "Dhyt5rqb-Elk"
    },
    {
      "cell_type": "code",
      "execution_count": 220,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "AfM_E-Xt-Gk1",
        "outputId": "6130d1dc-ea7f-4bef-e662-6dae8fb529e1"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEWCAYAAACEz/viAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAATH0lEQVR4nO3df5hVBZ3H8fcXIckkf4C5LaRgaWqkg0FhiMXjrqgkZm35o90IKyvbfdZ60qCetMjdaG3dNcXY2vyVv2g1WvexjHR9lGfLCHQUExFQtEEExSAoLdDv/nHP1HWaGWbonnNn4P16nnnm3HPPPfdzz/3xmXPOnXMiM5Ek7doGNDuAJKn5LANJkmUgSbIMJElYBpIkLANJEpaBJAnLQOqRiFgdEc9HxOaI2BgRP4mIj0eE7yHtFHwhSz13cmYOAQ4EZgOfBb7d3EhSY1gGUi9l5qbMvBU4DZgWEaMjYveI+FpEPBkR6yJibkS8EiAilkXEu9pvHxEDI+KZiDiqWY9B6sgykHZQZi4C2oCJ1NYUDgFagDcAw4ELiklvBM6ou+lk4NnMvK+6tFL3LAPpz/MUsC9wNvCpzHwuMzcD/wycXkxzAzA1IvYoLp9JrSCkPmNgswNI/dxwau+jPYAlEdE+PoDdADJzZUQsA06OiP8BpgJjmpBV6pJlIO2giBhHrQy+T21n8psyc00Xk7dvKhoAPJyZK6tJKfWMm4mkXoqIVxc7hG8CrsvMB4BvAf8WEa8pphkeEZPrbnYTcDzwCWqbjaQ+JTyfgbR9EbEa2B/YBrwEPAxcB8zNzBcjYjC1HcanA8OANcA3MvPrdfO4E3gHMCIzn672EUjdswwkSW4mkiRZBpIkLANJEpaBJIl++n8Gw4YNy5EjRzY7hiT1K0uWLHk2M/fr7Lp+WQYjR45k8eLFzY4hSf1KRDzR1XVuJpIkWQaSJMtAkkQ/3WcgSV3ZunUrbW1tvPDCC82O0jSDBw9mxIgRDBo0qMe3sQwk7VTa2toYMmQII0eOpO6Q4ruMzGTDhg20tbUxatSoHt/OzUSSdiovvPACQ4cO3SWLACAiGDp0aK/XjCwDSTudXbUI2u3I47cMJEnuM5C0cxs547aGzm/17CndXr9x40ZuuOEGzjnnnF7Pu7W1laeeeoqTTjppR+PtMNcMJKmBNm7cyBVXXLFDt21tbeUHP/hBgxP1jGUgSQ00Y8YMVq1aRUtLC+eddx4XX3wx48aN44gjjuDCCy8EYP78+Rx33HFkJmvXruWQQw7hySef5IILLmDevHm0tLQwb968Tue/aNEijj76aMaMGcPb3/52li9f3pDcbiaSpAaaPXs2Dz30EK2trSxYsICbb76ZRYsWkZlMnTqVe+65h1NPPZVbbrmFOXPmcPvtt/OlL32JAw44gFmzZrF48WIuv/zyLud/6KGHsnDhQgYOHMgdd9zB5z73OW655ZY/O7dlIEklWbBgAQsWLGDMmDEAbNmyhRUrVnDsscdy2WWXMXr0aMaPH88ZZ5zR43lu2rSJadOmsWLFCiKCrVu3NiSrZSBJJclMZs6cycc+9rE/ua6trY0BAwawbt06XnrpJQYM6NlW+y984QtMmjSJ+fPns3r1at75znc2JKv7DCSpgYYMGcLmzZsBmDx5MldeeSVbtmwBYM2aNaxfv55t27Zx1llnceONN3LYYYdxySWX/Mltu7Jp0yaGDx8OwNVXX92w3K4ZSNqpbe+roI02dOhQJkyYwOjRoznxxBM588wzOfroowHYc889ue6665g7dy4TJ07kmGOO4cgjj2TcuHFMmTKFSZMmMXv2bFpaWpg5cyannXban8z//PPPZ9q0aVx00UVMmdK4xxaZ2bCZVWXs2LHpyW0kdWbZsmUcdthhzY7RdJ0th4hYkpljO5vezUSSJDcTSVJfdNVVV3HppZe+bNyECROYM2dOKfdnGUhSHzR9+nSmT59e2f25mUjSTqc/7gttpB15/JaBpJ3K4MGD2bBhwy5bCO0ntxk8eHCvbudmIkk7lREjRtDW1sYzzzzT7ChN037ay96wDCTtVAYNGtSr0z2qxs1EkiTLQJLUTzcTLV2zqeFnL1J5qj4cgKTec81AkmQZSJIsA0kSloEkCctAkoRlIEnCMpAkYRlIkrAMJElYBpIkLANJEpaBJAnLQJKEZSBJwjKQJGEZSJKwDCRJWAaSJCwDSRKWgSQJy0CShGUgScIykCRhGUiSsAwkScDAMmYaEbOA5zLz34vL/wSsB14BvB/YHZifmRdGxKuA7wIjgN2AL2fmvDJySZI6V9aawZXABwEiYgBwOvA0cDDwVqAFeEtEHAucADyVmUdm5mjg9s5mGBFnR8TiiFj84m83lRRbknZNpZRBZq4GNkTEGOB44H5gXN3wfcCh1MphKfDXEfHViJiYmZ1+0mfmNzNzbGaO3W2PvcqILUm7rFI2ExX+E/gQ8BfU1hSOA76Smf/RccKIOAo4CbgoIu7MzFkl5pIkdVBmGcwHZgGDgDOBbcCXI+L6zNwSEcOBrUWG5zLzuojYCHykxEySpE6UVgaZ+fuIuAvYmJkvAgsi4jDgpxEBsAX4W+ANwMUR8RK1cvhEWZkkSZ0rrQyKHcfjgfe1j8vMS4FLO0y6CvhRWTkkSdtXyg7kiDgcWAncmZkryrgPSVLjlLJmkJkPAweVMW9JUuP5H8iSJMtAkmQZSJKwDCRJWAaSJCwDSRKWgSQJy0CShGUgScIykCRhGUiSsAwkSVgGkiQsA0kSloEkCctAkoRlIEnCMpAkYRlIkijpHMhle/PwvVg8e0qzY0jSTsM1A0mSZSBJsgwkSVgGkiQsA0kSloEkCctAkoRlIEnCMpAkYRlIkrAMJElYBpIkLANJEpaBJAnLQJKEZSBJwjKQJLGdM51FxHu6uz4zv9fYOJKkZtjeaS9P7ua6BCwDSdoJdFsGmTm9qiCSpObp0T6DiNg/Ir4dET8sLh8eER8uN5okqSo93YF8NfAj4C+Ly48C55YRSJJUvZ6WwbDM/C7wEkBmbgNeLC2VJKlSPS2D30TEUGo7jYmI8cCm0lJJkiq1vW8Ttfs0cCvw+oj4P2A/4G9KSyVJqlSPyiAz74uIdwBvBAJYnplbS00mSapMj8ogIgYD5wDHUNtUtDAi5mbmC2WGkyRVo6ebia4FNgOXFZfPBL4DvK+MUJKkavW0DEZn5uF1l++KiIfLCCRJql5Pv010X/ENIgAi4m3A4nIiSZKqtr0D1S2lto9gEPCTiHiyuHwg8Ej58SRJVdjeZqJ3VZJCktRU2ztQ3RP1lyPiNcDgUhNJkirX0wPVTY2IFcDjwN3AauCHJeaSJFWopzuQvwyMBx7NzFHAccC9paWSJFWqp2WwNTM3AAMiYkBm3gWMLTGXJKlCPf0/g40RsSdwD3B9RKwHflNeLElSlXq6ZnAK8DzwKeB2YBXdnxJTktSP9PRAdfVrAdeUlEWS1CTb+6ezzRTnMOh4FZCZ+epSUkmSKrW9/zMYUlUQSVLz9HSfgSRpJ2YZSJIsA0mSZSBJwjKQJGEZSJKwDCRJWAaSJCwDSRKWgSQJy0CShGUgSaLnJ7fpU5au2cTIGbc1O4YkVWr17Cmlzds1A0mSZSBJsgwkSVgGkiQsA0kSloEkCctAkoRlIEnCMpAkYRlIkrAMJElYBpIkLANJEpaBJAnLQJKEZSBJwjKQJGEZSJKwDCRJWAaSJCwDSRKWgSQJy0CShGUgScIykCRRcRlExMiIWBYR34qIX0TEgoh4ZUS0RMS9EfFgRMyPiH2qzCVJu7pmrBkcDMzJzDcBG4H3AtcCn83MI4ClwIUdbxQRZ0fE4ohY/OJvN1UaWJJ2ds0og8czs7UYXgK8Htg7M+8uxl0DHNvxRpn5zcwcm5ljd9tjr4qiStKuoRll8Lu64ReBvZuQQZJUpy/sQN4E/CoiJhaX/w64u5vpJUkNNrDZAQrTgLkRsQfwGDC9yXkkaZdSaRlk5mpgdN3lr9VdPb7KLJKkP+oLm4kkSU1mGUiSLANJkmUgScIykCRhGUiSsAwkSVgGkiQsA0kSloEkCctAkoRlIEnCMpAkYRlIkrAMJElYBpIkLANJEpaBJAnLQJKEZSBJwjKQJGEZSJKAgc0OsCPePHwvFs+e0uwYkrTTcM1AkmQZSJIsA0kSloEkCctAkoRlIEnCMpAkYRlIkrAMJElYBpIkLANJEpaBJAnLQJKEZSBJwjKQJGEZSJKwDCRJWAaSJCwDSRKWgSQJy0CShGUgScIykCRhGUiSsAwkSVgGkiQgMrPZGXotIjYDy5udowvDgGebHaIT5uodc/WOuXqnWbkOzMz9OrtiYNVJGmR5Zo5tdojORMTivpjNXL1jrt4xV+/0xVxuJpIkWQaSpP5bBt9sdoBu9NVs5uodc/WOuXqnz+XqlzuQJUmN1V/XDCRJDWQZSJL6XxlExAkRsTwiVkbEjIrv+3URcVdEPBwRv4iIfyzGfzEi1kREa/FzUt1tZhZZl0fE5BKzrY6IpcX9Ly7G7RsRP46IFcXvfYrxERFfL3I9GBFHlZTpjXXLpDUifh0R5zZjeUXElRGxPiIeqhvX6+UTEdOK6VdExLSScl0cEY8U9z0/IvYuxo+MiOfrltvcutu8pXj+VxbZo4RcvX7eGv1+7SLXvLpMqyOitRhf5fLq6rOh6a+xHsvMfvMD7AasAg4CXgE8ABxe4f2/FjiqGB4CPAocDnwR+Ewn0x9eZNwdGFVk362kbKuBYR3G/QswoxieAXy1GD4J+CEQwHjgZxU9d08DBzZjeQHHAkcBD+3o8gH2BR4rfu9TDO9TQq7jgYHF8Ffrco2sn67DfBYVWaPIfmIJuXr1vJXxfu0sV4fr/xW4oAnLq6vPhqa/xnr609/WDN4KrMzMxzLz98BNwClV3Xlmrs3M+4rhzcAyYHg3NzkFuCkzf5eZjwMrqT2GqpwCXFMMXwO8u278tVlzL7B3RLy25CzHAasy84lupilteWXmPcBzndxfb5bPZODHmflcZv4K+DFwQqNzZeaCzNxWXLwXGNHdPIpsr87Me7P2iXJt3WNpWK5udPW8Nfz92l2u4q/79wM3djePkpZXV58NTX+N9VR/K4PhwC/rLrfR/YdxaSJiJDAG+Fkx6u+L1b0r21cFqTZvAgsiYklEnF2M2z8z1xbDTwP7NyFXu9N5+Zu02csLer98mrHczqL2F2S7URFxf0TcHRETi3HDiyxV5OrN81b18poIrMvMFXXjKl9eHT4b+sNrDOh/ZdAnRMSewC3AuZn5a+AbwOuBFmAttVXVqh2TmUcBJwKfjIhj668s/gJqyveII+IVwFTgv4pRfWF5vUwzl09XIuLzwDbg+mLUWuCAzBwDfBq4ISJeXWGkPve8dXAGL/+Do/Ll1clnwx/0xddYvf5WBmuA19VdHlGMq0xEDKL2ZF+fmd8DyMx1mfliZr4EfIs/btqoLG9mril+rwfmFxnWtW/+KX6vrzpX4UTgvsxcV2Rs+vIq9Hb5VJYvIj4EvAv4QPEhQrEZZkMxvITa9vhDigz1m5JKybUDz1uVy2sg8B5gXl3eSpdXZ58N9OHXWEf9rQx+DhwcEaOKvzZPB26t6s6LbZLfBpZl5iV14+u3t58KtH/T4Vbg9IjYPSJGAQdT23HV6Fyviogh7cPUdkA+VNx/+7cRpgH/XZfrg8U3GsYDm+pWZcvwsr/Ymr286vR2+fwIOD4i9ik2kRxfjGuoiDgBOB+Ympm/rRu/X0TsVgwfRG35PFZk+3VEjC9eox+seyyNzNXb563K9+tfAY9k5h82/1S5vLr6bKCPvsY6VcVe6kb+UNsL/yi1lv98xfd9DLXVvAeB1uLnJOA7wNJi/K3Aa+tu8/ki63L+zG8sdJPrIGrf1HgA+EX7cgGGAncCK4A7gH2L8QHMKXItBcaWuMxeBWwA9qobV/nyolZGa4Gt1LbDfnhHlg+1bfgri5/pJeVaSW27cftrbG4x7XuL57cVuA84uW4+Y6l9OK8CLqc4ukCDc/X6eWv0+7WzXMX4q4GPd5i2yuXV1WdD019jPf3xcBSSpH63mUiSVALLQJJkGUiSLANJEpaBJAnLQOqRiPhKREyKiHdHxMxi3NUR8XjUjoj5QEQc1+yc0o6yDKSeeRu1g8a9A7inbvx5mdkCnAvM7eyGUn9gGUjdiNq5BR4ExgE/BT4CfCMiLugw6U8pDigWEYMj4qqoHS///oiYVIy/LSKOKIbvb59HRMyKiI9W9ZikzlgGUjcy8zxq/317NbVCeDAzj8jMWR0mPQH4fjH8ydpN883UDsVxTUQMBhYCEyNiL2oHoJtQTD+Rl69tSJWzDKTtO4raoT4OpXac+noXR8SjwA3UTkQDtUMTXAeQmY8AT1A7QNpCaidnmQDcBuwZEXsAozJzedkPQurOwGYHkPqqiGihtkYwAngW2KM2OlqBo4vJzsvMmyPiH4Argbd0M8ufUzsmzmPUTloyDPgosKSUByD1gmsGUhcys7XYOdx+CsP/BSZnZktmPt9h8suBAVE7/+9C4AMAEXEIcACwPGtn+/ol8D5q+xgWAp/BTUTqAywDqRsRsR/wq6wdw//QzHy4s+mydsTHi6gdevoKasWwlNrx9T+Umb8rJl0IrC/KZCG1tY6FJT8Mabs8aqkkyTUDSZJlIEnCMpAkYRlIkrAMJElYBpIkLANJEvD/VDM1B1nmS/UAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "dev.groupby(['label']).count().plot(kind='barh')\n",
        "plt.title('Dev')\n",
        "plt.xlabel('#Row')\n",
        "plt.show()"
      ],
      "id": "AfM_E-Xt-Gk1"
    },
    {
      "cell_type": "code",
      "execution_count": 221,
      "metadata": {
        "id": "bvYQLmn2GQqs"
      },
      "outputs": [],
      "source": [
        "def evaluation(y_test, y_pred):\n",
        "  # Model Accuracy\n",
        "  print(\"Accuracy\\t:\",accuracy_score(y_test, y_pred))\n",
        "  # # Model Precision\n",
        "  # print(\"Precision\\t:\",precision_score(y_test, y_pred,average=\"binary\", pos_label=\"yes\"))\n",
        "  # # Model Recall\n",
        "  # print(\"Recall\\t\\t:\",recall_score(y_test, y_pred,average=\"binary\", pos_label=\"yes\"))\n",
        "  # Model F1\n",
        "  print(\"F1\\t\\t:\", f1_score(y_test, y_pred,average=\"binary\", pos_label=\"yes\"))\n",
        "\n",
        "  # Confusion Matrix\n",
        "  print(\"Confusion Matrix:\\n\", confusion_matrix(y_test, y_pred))\n",
        "  # Classification Report\n",
        "  print(\"Classification Report:\\n\", classification_report(y_test, y_pred))"
      ],
      "id": "bvYQLmn2GQqs"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e52abd6d"
      },
      "source": [
        "# Preprocessing"
      ],
      "id": "e52abd6d"
    },
    {
      "cell_type": "code",
      "execution_count": 222,
      "metadata": {
        "collapsed": true,
        "id": "866c40a0"
      },
      "outputs": [],
      "source": [
        "# Fungsi untuk membersihkan teks\n",
        "def clean_data(text):\n",
        "\n",
        "    # lowercase\n",
        "    cleaned_text = text.lower()\n",
        "    # hapus RT\n",
        "    cleaned_text = re.sub(r'^rt ', '', cleaned_text)\n",
        "    # hapus emoji\n",
        "    cleaned_text = re.sub(r'\\\\x.{2}', '', cleaned_text)\n",
        "    # hapus www.* atau https?://* (URL)\n",
        "    cleaned_text = re.sub(r'((www\\.[^\\s]*)|(https?://[^\\s]*))', '', cleaned_text)\n",
        "    # trim depan belakang\n",
        "    cleaned_text = cleaned_text.strip()\n",
        "    # hapus @username\n",
        "    cleaned_text = re.sub(r'@[^\\s]+', '', cleaned_text)\n",
        "    # hapus hashtag\n",
        "    cleaned_text = re.sub(r'#[^\\s]+', '', cleaned_text)\n",
        "    # hapus angka\n",
        "    # normal_tw = re.sub(r'\\d+', ' ', normal_tw) \n",
        "\n",
        "    # hapus nan\n",
        "    cleaned_text = re.sub(r'^nan$', '', cleaned_text) \n",
        "    # hapus underscore\n",
        "    cleaned_text = re.sub(r'[_]+', '', cleaned_text)\n",
        "    # hapus undefine letter\n",
        "    cleaned_text =  re.sub(r'[Ã°Âã¯¹¢²ðâ]', '', cleaned_text) \n",
        "    # regex huruf yang berulang kaya haiiii (untuk fitur unigram)\n",
        "    normal_regex = re.compile(r\"(.)\\1{1,}\")\n",
        "    # buang huruf yang berulang\n",
        "    cleaned_text = normal_regex.sub(r\"\\1\\1\", cleaned_text)\n",
        "    # remove spasi berlebih (whitespace character)\n",
        "    cleaned_text = re.sub(r'\\s+', ' ', cleaned_text)\n",
        "    # hapus tanda baca dan emoji\n",
        "    cleaned_text = re.sub(r'[^\\w\\s]', '', cleaned_text) \n",
        "    # hapus kata aneh\n",
        "    cleaned_text = re.sub(r'[^\\x00-\\x7F\\x80-\\xFF\\u0100-\\u017F\\u0180-\\u024F\\u1E00-\\u1EFF]', '', cleaned_text)\n",
        "    \n",
        "    return cleaned_text"
      ],
      "id": "866c40a0"
    },
    {
      "cell_type": "code",
      "execution_count": 223,
      "metadata": {
        "id": "86c5fe49"
      },
      "outputs": [],
      "source": [
        "# To get list of stopwords (bahasa Indonesia)\n",
        "resource_path = \"/content/drive/Shared drives/MLL + NLP/Text Classification NLP/\"\n",
        "stopwords_list = []\n",
        "with open(resource_path+\"stopwords.txt\", \"r\") as file:\n",
        "    for line in file:\n",
        "        stripped_line = line.strip()\n",
        "        stopwords_list.append(stripped_line)"
      ],
      "id": "86c5fe49"
    },
    {
      "cell_type": "code",
      "execution_count": 224,
      "metadata": {
        "id": "50eacfd2"
      },
      "outputs": [],
      "source": [
        "def remove_stopwords(text):\n",
        "    token = nltk.word_tokenize(text)\n",
        "    token_afterremoval = []\n",
        "    for k in token:\n",
        "        if k not in stopwords_list: \n",
        "            token_afterremoval.append(k)\n",
        "\n",
        "    str_clean = ' '.join(token_afterremoval)\n",
        "    return str_clean"
      ],
      "id": "50eacfd2"
    },
    {
      "cell_type": "code",
      "execution_count": 225,
      "metadata": {
        "id": "s9TMofrJIKsV"
      },
      "outputs": [],
      "source": [
        "from Sastrawi.Stemmer.StemmerFactory import StemmerFactory\n",
        "factory = StemmerFactory()\n",
        "stemmer = factory.create_stemmer()\n",
        "def stemming(text):\n",
        "    token = nltk.word_tokenize(text)\n",
        "    stem_kalimat = []\n",
        "    for k in token:\n",
        "        stem_kata = stemmer.stem(k)\n",
        "        stem_kalimat.append(stem_kata)\n",
        "    stem_kalimat_str = ' '.join(stem_kalimat)\n",
        "    return stem_kalimat_str"
      ],
      "id": "s9TMofrJIKsV"
    },
    {
      "cell_type": "code",
      "execution_count": 226,
      "metadata": {
        "id": "ysBqmlbe2X_U"
      },
      "outputs": [],
      "source": [
        "resource_path = \"/content/drive/Shared drives/MLL + NLP/Text Classification NLP/\"\n",
        "\n",
        "kamus_alay = pd.read_csv(resource_path+\"new_kamusalay.csv\",encoding=\"Latin-1\",names=['alay','baku'])\n",
        "\n",
        "kamus_alay_map = dict(zip(kamus_alay['alay'], kamus_alay['baku']))\n",
        "\n",
        "def normalize(text):\n",
        "    return ' '.join([kamus_alay_map[word] if word in kamus_alay_map else word for word in text.split(' ')])"
      ],
      "id": "ysBqmlbe2X_U"
    },
    {
      "cell_type": "code",
      "execution_count": 227,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v_7lCYEkzZht",
        "outputId": "b0958883-d18b-476c-ad73-ad315be3ded3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "mukenahhh tlongggg aku mau handphone aku kentank banget sudah belah hadiah ulang tahun hshs aku depok by the way ikut ya\n"
          ]
        }
      ],
      "source": [
        "print(stemming(normalize(\"mukenahhh tlongggg ak maw hp ak kentank bingits sdh kebelah hadiah ultah hshs ak depok btw follback yh\")))"
      ],
      "id": "v_7lCYEkzZht"
    },
    {
      "cell_type": "code",
      "execution_count": 228,
      "metadata": {
        "id": "be919e3a"
      },
      "outputs": [],
      "source": [
        "# Fungsi untuk melakukan preprocessing pada data \n",
        "def preprocess(text):\n",
        "    # Membersihkan data\n",
        "    processed_text = clean_data(text)\n",
        "    # Convert bahasa tidak baku ke bahasa baku\n",
        "    processed_text = normalize(processed_text)\n",
        "    # Menghilangkan stopwords\n",
        "    processed_text = remove_stopwords(processed_text)\n",
        "    return processed_text"
      ],
      "id": "be919e3a"
    },
    {
      "cell_type": "code",
      "execution_count": 229,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4XrGgtZwIkXC",
        "outputId": "d3e2b988-bf2b-4128-a567-77415f84b786"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "mukenahh tlongg handphone kentank banget kebelah hadiah ulang hshs depok by the way ikuti ya\n"
          ]
        }
      ],
      "source": [
        "print(preprocess(\"mukenahhh tlongggg ak maw hp ak kentank bingits sdh kebelah hadiah ultah hshs ak depok btw follback yh\"))"
      ],
      "id": "4XrGgtZwIkXC"
    },
    {
      "cell_type": "code",
      "execution_count": 230,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "df47ca0c",
        "outputId": "2c672b4b-18ea-4ffc-8f78-fca56fdee395"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Before\t: betewe buka twitter cuman ngetweet liat home berita corona panik kepikiran ndamau buka2 home yg aware aja i ll stay at home nda rumah kalo nda penting2 banget\n",
            "After\t: betewe buka twitter mentwit lihat home berita corona panik kepikiran ndamau buka buka home aware i ll stay at home rumah penting2 banget\n",
            "Before\t: mas piyuuu mugo2 corona tuh mulut tersumpal ma corona\n",
            "After\t: mas piyuu mugo2 corona mulut tersumpal corona\n",
            "Before\t: e100ss gini buka informasi sejelas nya identitas daerah penderita terjangkit infokan masyarakat mengisolasi nya kontak langsung penderita positif corona ditutup tutupi\n",
            "After\t: e100ss buka informasi sejelas nya identitas daerah penderita terjangkit infokan masyarakat mengisolasi nya kontak langsung penderita positif corona ditutup menutupi\n",
            "Before\t: neng solo wes ono terduga corona cobo neng ati mu neng conora\n",
            "After\t: neng solo ono terduga corona cobo neng hati mu neng conora\n",
            "Before\t: midiahn nii akun gak takut takut nya isu corona wkwkwkw\n",
            "After\t: midiahn akun takut takut nya isu corona wkwk\n"
          ]
        }
      ],
      "source": [
        "for text in train['text_a'][:5]:\n",
        "    print(\"Before\\t:\",text)\n",
        "    print(\"After\\t:\",preprocess(text))"
      ],
      "id": "df47ca0c"
    },
    {
      "cell_type": "code",
      "execution_count": 231,
      "metadata": {
        "id": "Izvh_orV_4Iv"
      },
      "outputs": [],
      "source": [
        "train['text_a'] = train.apply(lambda x: preprocess(x['text_a']), axis=1)"
      ],
      "id": "Izvh_orV_4Iv"
    },
    {
      "cell_type": "code",
      "execution_count": 232,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "x5uhv4QOB5Bg",
        "outputId": "32a66723-de66-4fca-833e-44ac85c9268e"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  text_a label\n",
              "0      betewe buka twitter mentwit lihat home berita ...    no\n",
              "1          mas piyuu mugo2 corona mulut tersumpal corona    no\n",
              "2      e100ss buka informasi sejelas nya identitas da...   yes\n",
              "3      neng solo ono terduga corona cobo neng hati mu...    no\n",
              "4           midiahn akun takut takut nya isu corona wkwk    no\n",
              "...                                                  ...   ...\n",
              "21596      depok panas karuan kereta pasming hujan deras    no\n",
              "21597  oxfara arie kriting goblok nya orang kebanyaka...    no\n",
              "21598  virus corona menyaba depok cuci tangan makan n...    no\n",
              "21599  mata sipit tinggal depok habis bahan cengan di...    no\n",
              "21600             i batuk pilek pusing demam anjir depok    no\n",
              "\n",
              "[21601 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4e936b10-cb79-4dc5-ac99-d30fe4c617cb\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text_a</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>betewe buka twitter mentwit lihat home berita ...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>mas piyuu mugo2 corona mulut tersumpal corona</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>e100ss buka informasi sejelas nya identitas da...</td>\n",
              "      <td>yes</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>neng solo ono terduga corona cobo neng hati mu...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>midiahn akun takut takut nya isu corona wkwk</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21596</th>\n",
              "      <td>depok panas karuan kereta pasming hujan deras</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21597</th>\n",
              "      <td>oxfara arie kriting goblok nya orang kebanyaka...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21598</th>\n",
              "      <td>virus corona menyaba depok cuci tangan makan n...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21599</th>\n",
              "      <td>mata sipit tinggal depok habis bahan cengan di...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21600</th>\n",
              "      <td>i batuk pilek pusing demam anjir depok</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>21601 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4e936b10-cb79-4dc5-ac99-d30fe4c617cb')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4e936b10-cb79-4dc5-ac99-d30fe4c617cb button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4e936b10-cb79-4dc5-ac99-d30fe4c617cb');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 232
        }
      ],
      "source": [
        "train"
      ],
      "id": "x5uhv4QOB5Bg"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9e8cb95e"
      },
      "source": [
        "# Modeling"
      ],
      "id": "9e8cb95e"
    },
    {
      "cell_type": "code",
      "execution_count": 233,
      "metadata": {
        "id": "NZU9ybRXYmwS"
      },
      "outputs": [],
      "source": [
        "x_train = train['text_a']\n",
        "y_train = train['label']\n",
        "x_test = test['text_a']\n",
        "y_test = test['label']\n",
        "x_val = dev['text_a']\n",
        "y_val = dev['label']"
      ],
      "id": "NZU9ybRXYmwS"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vi2JhQmzx2zJ"
      },
      "source": [
        "\n",
        "## Logistic Regression"
      ],
      "id": "Vi2JhQmzx2zJ"
    },
    {
      "cell_type": "code",
      "execution_count": 234,
      "metadata": {
        "id": "0X7YLh2bx1hr"
      },
      "outputs": [],
      "source": [
        "lr = LogisticRegression(random_state=42)\n",
        "token = RegexpTokenizer(r'[a-zA-Z0-9]+')\n",
        "cv = CountVectorizer(lowercase=True,ngram_range=(1,1),tokenizer=token.tokenize)\n",
        "lr_clf = Pipeline([('vect', cv),\n",
        "                     ('tfidf', TfidfTransformer()),\n",
        "                    ('clf', lr),\n",
        "])"
      ],
      "id": "0X7YLh2bx1hr"
    },
    {
      "cell_type": "code",
      "execution_count": 235,
      "metadata": {
        "id": "ot3L9FKBybWR"
      },
      "outputs": [],
      "source": [
        "lr_clf = lr_clf.fit(x_train,y_train)"
      ],
      "id": "ot3L9FKBybWR"
    },
    {
      "cell_type": "code",
      "execution_count": 236,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z80YYO3oyZOE",
        "outputId": "b2fca0db-a7e1-49b5-ada8-004b6ba9b128"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy\t: 0.8446428571428571\n",
            "F1\t\t: 0.6500402252614642\n",
            "Confusion Matrix:\n",
            " [[1961  303]\n",
            " [ 132  404]]\n",
            "Classification Report:\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "          no       0.94      0.87      0.90      2264\n",
            "         yes       0.57      0.75      0.65       536\n",
            "\n",
            "    accuracy                           0.84      2800\n",
            "   macro avg       0.75      0.81      0.78      2800\n",
            "weighted avg       0.87      0.84      0.85      2800\n",
            "\n"
          ]
        }
      ],
      "source": [
        "lr_pred = lr_clf.predict(x_test)\n",
        "evaluation(lr_pred,y_test)"
      ],
      "id": "Z80YYO3oyZOE"
    },
    {
      "cell_type": "code",
      "source": [
        "param_lr = {\n",
        "      # 'penalty': ['l1', 'l2'],\n",
        "      'C'      : np.logspace(-3,3,7),\n",
        "      'solver' : ['newton-cg', 'lbfgs', 'liblinear']\n",
        "}\n",
        "\n",
        "lr_grid = GridSearchCV(lr, \n",
        "                      param_grid = param_lr,\n",
        "                      scoring='accuracy',\n",
        "                      cv=5,\n",
        "                       verbose=1)\n",
        "lr_tuning = Pipeline([('vect', cv),\n",
        "                     ('tfidf', TfidfTransformer()),\n",
        "                    ('gridsearch', lr_grid),\n",
        "])\n",
        "\n",
        "# fitting the model for grid search\n",
        "lr_tuning.fit(x_train, y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C9izZJ6uTW6F",
        "outputId": "39aec2cd-501c-492b-bf29-be1c14a87f10"
      },
      "id": "C9izZJ6uTW6F",
      "execution_count": 237,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 5 folds for each of 21 candidates, totalling 105 fits\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(steps=[('vect',\n",
              "                 CountVectorizer(tokenizer=<bound method RegexpTokenizer.tokenize of RegexpTokenizer(pattern='[a-zA-Z0-9]+', gaps=False, discard_empty=True, flags=<RegexFlag.UNICODE|DOTALL|MULTILINE: 56>)>)),\n",
              "                ('tfidf', TfidfTransformer()),\n",
              "                ('gridsearch',\n",
              "                 GridSearchCV(cv=5,\n",
              "                              estimator=LogisticRegression(random_state=42),\n",
              "                              param_grid={'C': array([1.e-03, 1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02, 1.e+03]),\n",
              "                                          'solver': ['newton-cg', 'lbfgs',\n",
              "                                                     'liblinear']},\n",
              "                              scoring='accuracy', verbose=1))])"
            ]
          },
          "metadata": {},
          "execution_count": 237
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# best parameter after tuning\n",
        "print(lr_grid.best_params_)\n",
        " \n",
        "# Model terbaik setelah hyper-parameter tuning\n",
        "print(lr_grid.best_estimator_)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lQ4R4g9eTx8L",
        "outputId": "5d9e3a78-3a23-48a3-fdfd-8c87b32451ee"
      },
      "id": "lQ4R4g9eTx8L",
      "execution_count": 238,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'C': 100.0, 'solver': 'lbfgs'}\n",
            "LogisticRegression(C=100.0, random_state=42)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "token = RegexpTokenizer(r'[a-zA-Z0-9]+')\n",
        "cv = CountVectorizer(lowercase=True,ngram_range=(1,1),tokenizer=token.tokenize)\n",
        "lr_clf_tuned = Pipeline([('vect', cv),\n",
        "                     ('tfidf', TfidfTransformer()),\n",
        "                    ('clf', LogisticRegression(random_state=42,C=100.0,solver='lbfgs')),\n",
        "])"
      ],
      "metadata": {
        "id": "rFLZdeVLT00Z"
      },
      "id": "rFLZdeVLT00Z",
      "execution_count": 239,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit\n",
        "lr_clf_tuned = lr_clf_tuned.fit(x_train, y_train)"
      ],
      "metadata": {
        "id": "FattVxO0T3Ds"
      },
      "id": "FattVxO0T3Ds",
      "execution_count": 240,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lr_tuned_pred = lr_clf_tuned.predict(x_test)\n",
        "# predicted\n",
        "evaluation(lr_tuned_pred,y_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T2pHW4PGT5NK",
        "outputId": "ff95cd35-26a8-4b71-bfcf-6c8077d6217d"
      },
      "id": "T2pHW4PGT5NK",
      "execution_count": 241,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy\t: 0.8275\n",
            "F1\t\t: 0.6634146341463415\n",
            "Confusion Matrix:\n",
            " [[1841  231]\n",
            " [ 252  476]]\n",
            "Classification Report:\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "          no       0.88      0.89      0.88      2072\n",
            "         yes       0.67      0.65      0.66       728\n",
            "\n",
            "    accuracy                           0.83      2800\n",
            "   macro avg       0.78      0.77      0.77      2800\n",
            "weighted avg       0.83      0.83      0.83      2800\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eIYXjnPWUWfE"
      },
      "source": [
        "## Decision Tree"
      ],
      "id": "eIYXjnPWUWfE"
    },
    {
      "cell_type": "code",
      "source": [
        "token = RegexpTokenizer(r'[a-zA-Z0-9]+')\n",
        "cv = CountVectorizer(lowercase=True,ngram_range=(1,1),tokenizer=token.tokenize)\n",
        "decision_tree_clf = Pipeline([('vect', cv),\n",
        "                     ('tfidf', TfidfTransformer()),\n",
        "                    ('cclf', DecisionTreeClassifier(random_state=42)),\n",
        "])\n",
        "decision_tree_clf.fit(x_train, y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EhtXuUvCpiY4",
        "outputId": "e65396d9-7b07-44a7-adc5-f2ace717238f"
      },
      "id": "EhtXuUvCpiY4",
      "execution_count": 278,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(steps=[('vect',\n",
              "                 CountVectorizer(tokenizer=<bound method RegexpTokenizer.tokenize of RegexpTokenizer(pattern='[a-zA-Z0-9]+', gaps=False, discard_empty=True, flags=<RegexFlag.UNICODE|DOTALL|MULTILINE: 56>)>)),\n",
              "                ('tfidf', TfidfTransformer()),\n",
              "                ('cclf', DecisionTreeClassifier(random_state=42))])"
            ]
          },
          "metadata": {},
          "execution_count": 278
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "decision_tree_pred = decision_tree_clf.predict(x_test)\n",
        "# predicted\n",
        "evaluation(decision_tree_pred,y_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qvSwTmmXpp6S",
        "outputId": "1f7a077c-638d-4c36-c478-641e3dcb42ed"
      },
      "id": "qvSwTmmXpp6S",
      "execution_count": 279,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy\t: 0.7667857142857143\n",
            "F1\t\t: 0.5230094959824689\n",
            "Confusion Matrix:\n",
            " [[1789  349]\n",
            " [ 304  358]]\n",
            "Classification Report:\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "          no       0.85      0.84      0.85      2138\n",
            "         yes       0.51      0.54      0.52       662\n",
            "\n",
            "    accuracy                           0.77      2800\n",
            "   macro avg       0.68      0.69      0.68      2800\n",
            "weighted avg       0.77      0.77      0.77      2800\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 242,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qWvoRGCQe-8Y",
        "outputId": "427e851b-4450-48bd-f510-4f3064085591"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 4 folds for each of 50 candidates, totalling 200 fits\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(steps=[('vect',\n",
              "                 CountVectorizer(tokenizer=<bound method RegexpTokenizer.tokenize of RegexpTokenizer(pattern='[a-zA-Z0-9]+', gaps=False, discard_empty=True, flags=<RegexFlag.UNICODE|DOTALL|MULTILINE: 56>)>)),\n",
              "                ('tfidf', TfidfTransformer()),\n",
              "                ('gridsearch',\n",
              "                 GridSearchCV(cv=4,\n",
              "                              estimator=DecisionTreeClassifier(random_state=42),\n",
              "                              n_jobs=-1,\n",
              "                              param_grid={'criterion': ['gini', 'entropy'],\n",
              "                                          'max_depth': [2, 3, 5, 10, 20],\n",
              "                                          'min_samples_leaf': [5, 10, 20, 50,\n",
              "                                                               100]},\n",
              "                              scoring='accuracy', verbose=3))])"
            ]
          },
          "metadata": {},
          "execution_count": 242
        }
      ],
      "source": [
        "# Mencari paramter terbaik\n",
        "# hyperparameter tuning C with GridSearchCV\n",
        "# defining parameter range\n",
        "param_decision_tree = {\n",
        "    'max_depth': [2, 3, 5, 10, 20],\n",
        "    'min_samples_leaf': [5, 10, 20, 50, 100],\n",
        "    'criterion': [\"gini\", \"entropy\"]\n",
        "}\n",
        "decision_tree_grid = GridSearchCV(DecisionTreeClassifier(random_state=42), param_decision_tree,\n",
        "                    cv=4, n_jobs=-1, verbose=3, scoring = \"accuracy\")\n",
        " \n",
        "token = RegexpTokenizer(r'[a-zA-Z0-9]+')\n",
        "cv = CountVectorizer(lowercase=True,ngram_range=(1,1),tokenizer=token.tokenize)\n",
        "decision_tree_tuning = Pipeline([('vect', cv),\n",
        "                     ('tfidf', TfidfTransformer()),\n",
        "                    ('gridsearch', decision_tree_grid),\n",
        "])\n",
        "# fitting the model for grid search\n",
        "decision_tree_tuning.fit(x_train, y_train)"
      ],
      "id": "qWvoRGCQe-8Y"
    },
    {
      "cell_type": "code",
      "execution_count": 243,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MisKcj1wgPKS",
        "outputId": "d2b5fd98-b1f1-40de-f0b5-1c51a6d6ed8b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'criterion': 'gini', 'max_depth': 20, 'min_samples_leaf': 5}\n",
            "DecisionTreeClassifier(max_depth=20, min_samples_leaf=5, random_state=42)\n"
          ]
        }
      ],
      "source": [
        "# best parameter after tuning\n",
        "print(decision_tree_grid.best_params_)\n",
        " \n",
        "# Model terbaik setelah hyper-parameter tuning\n",
        "print(decision_tree_grid.best_estimator_)"
      ],
      "id": "MisKcj1wgPKS"
    },
    {
      "cell_type": "code",
      "execution_count": 244,
      "metadata": {
        "id": "85bQ5c5jTPub"
      },
      "outputs": [],
      "source": [
        "token = RegexpTokenizer(r'[a-zA-Z0-9]+')\n",
        "cv = CountVectorizer(lowercase=True,ngram_range=(1,1),tokenizer=token.tokenize)\n",
        "decision_tree_clf = Pipeline([('vect', cv),\n",
        "                     ('tfidf', TfidfTransformer()),\n",
        "                    ('clf', DecisionTreeClassifier(max_depth=20,min_samples_leaf=5,criterion='gini',random_state=42)),\n",
        "])"
      ],
      "id": "85bQ5c5jTPub"
    },
    {
      "cell_type": "code",
      "execution_count": 245,
      "metadata": {
        "id": "ZKr4_I_pTnB5"
      },
      "outputs": [],
      "source": [
        "# Fit\n",
        "decision_tree_clf = decision_tree_clf.fit(x_train, y_train)"
      ],
      "id": "ZKr4_I_pTnB5"
    },
    {
      "cell_type": "code",
      "execution_count": 246,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "foo0Yp-sTyvZ",
        "outputId": "7c81b84f-9c5d-4c54-a168-b7775f47d65b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy\t: 0.7925\n",
            "F1\t\t: 0.457516339869281\n",
            "Confusion Matrix:\n",
            " [[1974  462]\n",
            " [ 119  245]]\n",
            "Classification Report:\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "          no       0.94      0.81      0.87      2436\n",
            "         yes       0.35      0.67      0.46       364\n",
            "\n",
            "    accuracy                           0.79      2800\n",
            "   macro avg       0.64      0.74      0.66      2800\n",
            "weighted avg       0.87      0.79      0.82      2800\n",
            "\n"
          ]
        }
      ],
      "source": [
        "decision_tree_pred = decision_tree_clf.predict(x_test)\n",
        "# predicted\n",
        "evaluation(decision_tree_pred,y_test)"
      ],
      "id": "foo0Yp-sTyvZ"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-urWUbhMUTlV"
      },
      "source": [
        "## Naive Bayes"
      ],
      "id": "-urWUbhMUTlV"
    },
    {
      "cell_type": "code",
      "execution_count": 247,
      "metadata": {
        "id": "PYjq4ZRy-84k"
      },
      "outputs": [],
      "source": [
        "token = RegexpTokenizer(r'[a-zA-Z0-9]+')\n",
        "cv = CountVectorizer(lowercase=True,ngram_range=(1,1),tokenizer=token.tokenize)\n",
        "nb_clf = Pipeline([('vect', cv),\n",
        "                     ('tfidf', TfidfTransformer()),\n",
        "                    ('clf', MultinomialNB()),\n",
        "])"
      ],
      "id": "PYjq4ZRy-84k"
    },
    {
      "cell_type": "code",
      "execution_count": 248,
      "metadata": {
        "id": "K6R_kg6M_EEJ"
      },
      "outputs": [],
      "source": [
        "nb_clf = nb_clf.fit(x_train,y_train)"
      ],
      "id": "K6R_kg6M_EEJ"
    },
    {
      "cell_type": "code",
      "execution_count": 249,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zQmlmCy8_MxZ",
        "outputId": "4d7cb9a7-8965-4a48-c4cd-ce52dc0c19d3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy\t: 0.8332142857142857\n",
            "F1\t\t: 0.5461613216715258\n",
            "Confusion Matrix:\n",
            " [[2052  426]\n",
            " [  41  281]]\n",
            "Classification Report:\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "          no       0.98      0.83      0.90      2478\n",
            "         yes       0.40      0.87      0.55       322\n",
            "\n",
            "    accuracy                           0.83      2800\n",
            "   macro avg       0.69      0.85      0.72      2800\n",
            "weighted avg       0.91      0.83      0.86      2800\n",
            "\n"
          ]
        }
      ],
      "source": [
        "nb_pred = nb_clf.predict(x_test)\n",
        "# predicted\n",
        "evaluation(nb_pred,y_test)"
      ],
      "id": "zQmlmCy8_MxZ"
    },
    {
      "cell_type": "code",
      "execution_count": 250,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wPk1yuR09eMD",
        "outputId": "3c66977b-81c6-4c58-d235-0ec6b71042f7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy\t: 0.8332142857142857\n",
            "F1\t\t: 0.5461613216715258\n",
            "Confusion Matrix:\n",
            " [[2052  426]\n",
            " [  41  281]]\n",
            "Classification Report:\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "          no       0.98      0.83      0.90      2478\n",
            "         yes       0.40      0.87      0.55       322\n",
            "\n",
            "    accuracy                           0.83      2800\n",
            "   macro avg       0.69      0.85      0.72      2800\n",
            "weighted avg       0.91      0.83      0.86      2800\n",
            "\n"
          ]
        }
      ],
      "source": [
        "nb_pred_val = nb_clf.predict(x_val)\n",
        "# predicted\n",
        "evaluation(nb_pred_val,y_test)"
      ],
      "id": "wPk1yuR09eMD"
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "cv_method = RepeatedStratifiedKFold(n_splits=5, \n",
        "                                    n_repeats=3, \n",
        "                                    random_state=42)"
      ],
      "metadata": {
        "id": "DYIGyU--QmbX"
      },
      "id": "DYIGyU--QmbX",
      "execution_count": 251,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "params_NB = {'alpha': [i/10.0 for i in range(0,10)]}\n",
        "\n",
        "gs_NB = GridSearchCV(estimator=MultinomialNB(), \n",
        "                     param_grid=params_NB, \n",
        "                     cv=cv_method,\n",
        "                     verbose=1, \n",
        "                     scoring='accuracy')\n",
        "token = RegexpTokenizer(r'[a-zA-Z0-9]+')\n",
        "cv = CountVectorizer(lowercase=True,ngram_range=(1,1),tokenizer=token.tokenize)\n",
        "grid_search_NB = Pipeline([('vect', cv),\n",
        "                     ('tfidf', TfidfTransformer()),\n",
        "                    ('gridsearch', gs_NB),\n",
        "])\n",
        "# fitting the model for grid search\n",
        "grid_search_NB.fit(x_train, y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sSd0I3ZdQni9",
        "outputId": "4d811486-8ff8-4ec3-9a9f-cc80f784c1af"
      },
      "id": "sSd0I3ZdQni9",
      "execution_count": 252,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 15 folds for each of 10 candidates, totalling 150 fits\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(steps=[('vect',\n",
              "                 CountVectorizer(tokenizer=<bound method RegexpTokenizer.tokenize of RegexpTokenizer(pattern='[a-zA-Z0-9]+', gaps=False, discard_empty=True, flags=<RegexFlag.UNICODE|DOTALL|MULTILINE: 56>)>)),\n",
              "                ('tfidf', TfidfTransformer()),\n",
              "                ('gridsearch',\n",
              "                 GridSearchCV(cv=RepeatedStratifiedKFold(n_repeats=3, n_splits=5, random_state=42),\n",
              "                              estimator=MultinomialNB(),\n",
              "                              param_grid={'alpha': [0.0, 0.1, 0.2, 0.3, 0.4,\n",
              "                                                    0.5, 0.6, 0.7, 0.8, 0.9]},\n",
              "                              scoring='accuracy', verbose=1))])"
            ]
          },
          "metadata": {},
          "execution_count": 252
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# best parameter after tuning\n",
        "print(gs_NB.best_params_)\n",
        " \n",
        "# Model terbaik setelah hyper-parameter tuning\n",
        "print(gs_NB.best_estimator_)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "axcASjMmR36J",
        "outputId": "b87bf909-8d75-4733-e776-4df50aae3e8e"
      },
      "id": "axcASjMmR36J",
      "execution_count": 253,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'alpha': 0.5}\n",
            "MultinomialNB(alpha=0.5)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "token = RegexpTokenizer(r'[a-zA-Z0-9]+')\n",
        "cv = CountVectorizer(lowercase=True,ngram_range=(1,1),tokenizer=token.tokenize)\n",
        "nb_clf_tuned = Pipeline([('vect', cv),\n",
        "                     ('tfidf', TfidfTransformer()),\n",
        "                    ('clf', MultinomialNB(alpha=0.5)),\n",
        "])"
      ],
      "metadata": {
        "id": "TkjEYv1dSRVY"
      },
      "id": "TkjEYv1dSRVY",
      "execution_count": 254,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "nb_clf_tuned = nb_clf_tuned.fit(x_train,y_train)"
      ],
      "metadata": {
        "id": "anpi2KriSV22"
      },
      "id": "anpi2KriSV22",
      "execution_count": 255,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "nb_tuned_pred = nb_clf_tuned.predict(x_test)\n",
        "\n",
        "evaluation(nb_tuned_pred,y_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HTWZtSmvSZTQ",
        "outputId": "2bf30413-bc78-4adb-e022-8a4d2d42b607"
      },
      "id": "HTWZtSmvSZTQ",
      "execution_count": 256,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy\t: 0.8514285714285714\n",
            "F1\t\t: 0.630550621669627\n",
            "Confusion Matrix:\n",
            " [[2029  352]\n",
            " [  64  355]]\n",
            "Classification Report:\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "          no       0.97      0.85      0.91      2381\n",
            "         yes       0.50      0.85      0.63       419\n",
            "\n",
            "    accuracy                           0.85      2800\n",
            "   macro avg       0.74      0.85      0.77      2800\n",
            "weighted avg       0.90      0.85      0.87      2800\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VBQAfggjUYMi"
      },
      "source": [
        "## SVM"
      ],
      "id": "VBQAfggjUYMi"
    },
    {
      "cell_type": "code",
      "source": [
        "token = RegexpTokenizer(r'[a-zA-Z0-9]+')\n",
        "cv = CountVectorizer(lowercase=True,ngram_range=(1,1),tokenizer=token.tokenize)\n",
        "svm_clf = Pipeline([('vect', cv),\n",
        "                     ('tfidf', TfidfTransformer()),\n",
        "                    ('cclf', SVC(random_state=42)),\n",
        "])\n",
        "svm_clf.fit(x_train, y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rb01gjvH5qqu",
        "outputId": "fae266b4-9350-4798-eec7-a993216e5aec"
      },
      "execution_count": 280,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(steps=[('vect',\n",
              "                 CountVectorizer(tokenizer=<bound method RegexpTokenizer.tokenize of RegexpTokenizer(pattern='[a-zA-Z0-9]+', gaps=False, discard_empty=True, flags=<RegexFlag.UNICODE|DOTALL|MULTILINE: 56>)>)),\n",
              "                ('tfidf', TfidfTransformer()), ('cclf', SVC(random_state=42))])"
            ]
          },
          "metadata": {},
          "execution_count": 280
        }
      ],
      "id": "rb01gjvH5qqu"
    },
    {
      "cell_type": "code",
      "source": [
        "svm_pred = svm_clf.predict(x_test)\n",
        "# predicted\n",
        "evaluation(svm_pred,y_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WA_pbJCi5qqy",
        "outputId": "eea3a2dc-b6bd-4b25-a425-d87d5abee0b8"
      },
      "execution_count": 281,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy\t: 0.865\n",
            "F1\t\t: 0.7018927444794952\n",
            "Confusion Matrix:\n",
            " [[1977  262]\n",
            " [ 116  445]]\n",
            "Classification Report:\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "          no       0.94      0.88      0.91      2239\n",
            "         yes       0.63      0.79      0.70       561\n",
            "\n",
            "    accuracy                           0.86      2800\n",
            "   macro avg       0.79      0.84      0.81      2800\n",
            "weighted avg       0.88      0.86      0.87      2800\n",
            "\n"
          ]
        }
      ],
      "id": "WA_pbJCi5qqy"
    },
    {
      "cell_type": "code",
      "source": [
        "svm_pred_val = svm_clf.predict(x_val)\n",
        "# predicted\n",
        "evaluation(svm_pred_val,y_val)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3yUa420e60RP",
        "outputId": "308aa94b-28ce-40db-ceb8-41eee9c86e03"
      },
      "id": "3yUa420e60RP",
      "execution_count": 283,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy\t: 0.865\n",
            "F1\t\t: 0.7018927444794952\n",
            "Confusion Matrix:\n",
            " [[1977  262]\n",
            " [ 116  445]]\n",
            "Classification Report:\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "          no       0.94      0.88      0.91      2239\n",
            "         yes       0.63      0.79      0.70       561\n",
            "\n",
            "    accuracy                           0.86      2800\n",
            "   macro avg       0.79      0.84      0.81      2800\n",
            "weighted avg       0.88      0.86      0.87      2800\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 257,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0yGSNbhSUZl3",
        "outputId": "fc688703-b375-40a9-8a02-94596ec6fede"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 5 folds for each of 5 candidates, totalling 25 fits\n",
            "[CV 1/5] END C=0.1, kernel=linear, random_state=42;, score=0.832 total time=  47.0s\n",
            "[CV 2/5] END C=0.1, kernel=linear, random_state=42;, score=0.836 total time=  46.5s\n",
            "[CV 3/5] END C=0.1, kernel=linear, random_state=42;, score=0.834 total time=  36.8s\n",
            "[CV 4/5] END C=0.1, kernel=linear, random_state=42;, score=0.828 total time=  36.8s\n",
            "[CV 5/5] END C=0.1, kernel=linear, random_state=42;, score=0.820 total time=  39.0s\n",
            "[CV 1/5] END C=1, kernel=linear, random_state=42;, score=0.878 total time=  49.6s\n",
            "[CV 2/5] END C=1, kernel=linear, random_state=42;, score=0.857 total time=  50.2s\n",
            "[CV 3/5] END C=1, kernel=linear, random_state=42;, score=0.895 total time=  51.6s\n",
            "[CV 4/5] END C=1, kernel=linear, random_state=42;, score=0.875 total time=  51.5s\n",
            "[CV 5/5] END C=1, kernel=linear, random_state=42;, score=0.923 total time=  55.6s\n",
            "[CV 1/5] END C=10, kernel=linear, random_state=42;, score=0.875 total time= 2.1min\n",
            "[CV 2/5] END C=10, kernel=linear, random_state=42;, score=0.835 total time= 1.5min\n",
            "[CV 3/5] END C=10, kernel=linear, random_state=42;, score=0.903 total time= 1.7min\n",
            "[CV 4/5] END C=10, kernel=linear, random_state=42;, score=0.855 total time= 2.1min\n",
            "[CV 5/5] END C=10, kernel=linear, random_state=42;, score=0.969 total time= 2.1min\n",
            "[CV 1/5] END C=100, kernel=linear, random_state=42;, score=0.867 total time= 1.5min\n",
            "[CV 2/5] END C=100, kernel=linear, random_state=42;, score=0.832 total time= 1.2min\n",
            "[CV 3/5] END C=100, kernel=linear, random_state=42;, score=0.895 total time= 1.4min\n",
            "[CV 4/5] END C=100, kernel=linear, random_state=42;, score=0.854 total time= 1.4min\n",
            "[CV 5/5] END C=100, kernel=linear, random_state=42;, score=0.972 total time= 1.8min\n",
            "[CV 1/5] END C=1000, kernel=linear, random_state=42;, score=0.862 total time= 1.5min\n",
            "[CV 2/5] END C=1000, kernel=linear, random_state=42;, score=0.822 total time= 1.2min\n",
            "[CV 3/5] END C=1000, kernel=linear, random_state=42;, score=0.897 total time= 1.4min\n",
            "[CV 4/5] END C=1000, kernel=linear, random_state=42;, score=0.846 total time= 1.4min\n",
            "[CV 5/5] END C=1000, kernel=linear, random_state=42;, score=0.972 total time= 1.7min\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(steps=[('vect',\n",
              "                 CountVectorizer(tokenizer=<bound method RegexpTokenizer.tokenize of RegexpTokenizer(pattern='[a-zA-Z0-9]+', gaps=False, discard_empty=True, flags=<RegexFlag.UNICODE|DOTALL|MULTILINE: 56>)>)),\n",
              "                ('tfidf', TfidfTransformer()),\n",
              "                ('gridsearch',\n",
              "                 GridSearchCV(estimator=SVC(),\n",
              "                              param_grid={'C': [0.1, 1, 10, 100, 1000],\n",
              "                                          'kernel': ['linear'],\n",
              "                                          'random_state': [42]},\n",
              "                              scoring='accuracy', verbose=3))])"
            ]
          },
          "metadata": {},
          "execution_count": 257
        }
      ],
      "source": [
        "# Mencari paramter terbaik\n",
        "# hyperparameter tuning with GridSearchCV\n",
        "# defining parameter range\n",
        "param_grid = {'C': [0.1, 1, 10, 100, 1000],\n",
        "              'kernel': ['linear'], 'random_state':[42]}\n",
        " \n",
        "grid = GridSearchCV(SVC(), param_grid, refit = True, verbose = 3,scoring = \"accuracy\")\n",
        " \n",
        "token = RegexpTokenizer(r'[a-zA-Z0-9]+')\n",
        "cv = CountVectorizer(lowercase=True,ngram_range=(1,1),tokenizer=token.tokenize)\n",
        "grid_search = Pipeline([('vect', cv),\n",
        "                     ('tfidf', TfidfTransformer()),\n",
        "                    ('gridsearch', grid),\n",
        "])\n",
        "# fitting the model for grid search\n",
        "grid_search.fit(train['text_a'], train['label'])"
      ],
      "id": "0yGSNbhSUZl3"
    },
    {
      "cell_type": "code",
      "execution_count": 258,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7kBYTaAFU91O",
        "outputId": "87c3cdc6-28a7-4946-f15f-3a99b83b97a1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'C': 10, 'kernel': 'linear', 'random_state': 42}\n",
            "SVC(C=10, kernel='linear', random_state=42)\n"
          ]
        }
      ],
      "source": [
        "# best parameter after tuning\n",
        "print(grid.best_params_)\n",
        " \n",
        "# Model terbaik setelah hyper-parameter tuning\n",
        "print(grid.best_estimator_)"
      ],
      "id": "7kBYTaAFU91O"
    },
    {
      "cell_type": "code",
      "execution_count": 259,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M6YU6WQ7c1un",
        "outputId": "336fd1f5-d587-41e5-9497-170161fb8318"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(steps=[('vect',\n",
              "                 CountVectorizer(tokenizer=<bound method RegexpTokenizer.tokenize of RegexpTokenizer(pattern='[a-zA-Z0-9]+', gaps=False, discard_empty=True, flags=<RegexFlag.UNICODE|DOTALL|MULTILINE: 56>)>)),\n",
              "                ('tfidf', TfidfTransformer()),\n",
              "                ('clf', SVC(C=10, kernel='linear', random_state=42))])"
            ]
          },
          "metadata": {},
          "execution_count": 259
        }
      ],
      "source": [
        "svm = SVC(kernel='linear',C=10,random_state=42)\n",
        "svm_clf = Pipeline([('vect', cv),\n",
        "                     ('tfidf', TfidfTransformer()),\n",
        "                    ('clf', svm),\n",
        "])\n",
        "# Train the model using training sets\n",
        "svm_clf.fit(x_train, y_train)"
      ],
      "id": "M6YU6WQ7c1un"
    },
    {
      "cell_type": "code",
      "execution_count": 260,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KcdPscfydLq_",
        "outputId": "621b345d-c586-4b7a-b882-2e39e617b475"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy\t: 0.8035714285714286\n",
            "F1\t\t: 0.6400523560209426\n",
            "Confusion Matrix:\n",
            " [[1761  218]\n",
            " [ 332  489]]\n",
            "Classification Report:\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "          no       0.84      0.89      0.86      1979\n",
            "         yes       0.69      0.60      0.64       821\n",
            "\n",
            "    accuracy                           0.80      2800\n",
            "   macro avg       0.77      0.74      0.75      2800\n",
            "weighted avg       0.80      0.80      0.80      2800\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Predict\n",
        "svm_pred = svm_clf.predict(x_test)\n",
        "evaluation(svm_pred,y_test)"
      ],
      "id": "KcdPscfydLq_"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JdF8ITaW6vfS"
      },
      "source": [
        "## XGBoost"
      ],
      "id": "JdF8ITaW6vfS"
    },
    {
      "cell_type": "code",
      "execution_count": 261,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TUW-COaa6xAZ",
        "outputId": "0eee356b-5b88-4c3b-c6ef-05f4e7dd6edf"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(steps=[('vect',\n",
              "                 CountVectorizer(tokenizer=<bound method RegexpTokenizer.tokenize of RegexpTokenizer(pattern='[a-zA-Z0-9]+', gaps=False, discard_empty=True, flags=<RegexFlag.UNICODE|DOTALL|MULTILINE: 56>)>)),\n",
              "                ('tfidf', TfidfTransformer()),\n",
              "                ('clf',\n",
              "                 XGBClassifier(colsample_bytree=0.8, max_depth=5,\n",
              "                               n_estimators=1000, nthread=4, random_state=42,\n",
              "                               seed=27, subsample=0.8))])"
            ]
          },
          "metadata": {},
          "execution_count": 261
        }
      ],
      "source": [
        "xgb = XGBClassifier(random_state=42,\n",
        "                     learning_rate =0.1,\n",
        " n_estimators=1000,\n",
        " max_depth=5,\n",
        " min_child_weight=1,\n",
        " gamma=0,\n",
        " subsample=0.8,\n",
        " colsample_bytree=0.8,\n",
        " objective= 'binary:logistic',\n",
        " nthread=4,\n",
        " scale_pos_weight=1,\n",
        " seed=27)\n",
        "xgb_clf = Pipeline([('vect', cv),\n",
        "                     ('tfidf', TfidfTransformer()),\n",
        "                    ('clf', xgb),\n",
        "])\n",
        "# Train the model using training sets\n",
        "xgb_clf.fit(x_train, y_train)"
      ],
      "id": "TUW-COaa6xAZ"
    },
    {
      "cell_type": "code",
      "execution_count": 262,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fQ02BkZ07gnt",
        "outputId": "a18507f0-363c-4622-a743-f3a95ee8f022"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy\t: 0.8410714285714286\n",
            "F1\t\t: 0.6402586903799514\n",
            "Confusion Matrix:\n",
            " [[1959  311]\n",
            " [ 134  396]]\n",
            "Classification Report:\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "          no       0.94      0.86      0.90      2270\n",
            "         yes       0.56      0.75      0.64       530\n",
            "\n",
            "    accuracy                           0.84      2800\n",
            "   macro avg       0.75      0.81      0.77      2800\n",
            "weighted avg       0.86      0.84      0.85      2800\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Predict\n",
        "xgb_pred = xgb_clf.predict(x_test)\n",
        "evaluation(xgb_pred,y_test)"
      ],
      "id": "fQ02BkZ07gnt"
    },
    {
      "cell_type": "code",
      "execution_count": 263,
      "metadata": {
        "id": "2xqxY4SF-Ngb"
      },
      "outputs": [],
      "source": [
        "param_test1 = {\n",
        " 'max_depth':range(3,10,2),\n",
        " 'min_child_weight':range(1,6,2)\n",
        "}\n",
        "gsearch1 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=140, max_depth=5,\n",
        " min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.8,\n",
        " objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27), \n",
        " param_grid = param_test1,n_jobs=4, cv=5)"
      ],
      "id": "2xqxY4SF-Ngb"
    },
    {
      "cell_type": "code",
      "execution_count": 264,
      "metadata": {
        "id": "yd-J-Yjo-7HP"
      },
      "outputs": [],
      "source": [
        "xgb_tuning = Pipeline([('vect', cv),\n",
        "                     ('tfidf', TfidfTransformer()),\n",
        "                    ('gridsearch', gsearch1),\n",
        "])"
      ],
      "id": "yd-J-Yjo-7HP"
    },
    {
      "cell_type": "code",
      "execution_count": 265,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_nJ4E1JQ-9bS",
        "outputId": "4053fdb1-e330-443c-a4ff-99c549434a2f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(steps=[('vect',\n",
              "                 CountVectorizer(tokenizer=<bound method RegexpTokenizer.tokenize of RegexpTokenizer(pattern='[a-zA-Z0-9]+', gaps=False, discard_empty=True, flags=<RegexFlag.UNICODE|DOTALL|MULTILINE: 56>)>)),\n",
              "                ('tfidf', TfidfTransformer()),\n",
              "                ('gridsearch',\n",
              "                 GridSearchCV(cv=5,\n",
              "                              estimator=XGBClassifier(colsample_bytree=0.8,\n",
              "                                                      max_depth=5,\n",
              "                                                      n_estimators=140,\n",
              "                                                      nthread=4, seed=27,\n",
              "                                                      subsample=0.8),\n",
              "                              n_jobs=4,\n",
              "                              param_grid={'max_depth': range(3, 10, 2),\n",
              "                                          'min_child_weight': range(1, 6, 2)}))])"
            ]
          },
          "metadata": {},
          "execution_count": 265
        }
      ],
      "source": [
        "xgb_tuning.fit(x_train,y_train)"
      ],
      "id": "_nJ4E1JQ-9bS"
    },
    {
      "cell_type": "code",
      "execution_count": 266,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1BXmREDHDTha",
        "outputId": "a51ec21d-d927-4a88-80be-0fb45168ae4d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'max_depth': 9, 'min_child_weight': 1}\n",
            "XGBClassifier(colsample_bytree=0.8, max_depth=9, n_estimators=140, nthread=4,\n",
            "              seed=27, subsample=0.8)\n"
          ]
        }
      ],
      "source": [
        "# best parameter after tuning\n",
        "print(gsearch1.best_params_)\n",
        " \n",
        "# Model terbaik setelah hyper-parameter tuning\n",
        "print(gsearch1.best_estimator_)"
      ],
      "id": "1BXmREDHDTha"
    },
    {
      "cell_type": "code",
      "execution_count": 267,
      "metadata": {
        "id": "80JDvFHzEIWL"
      },
      "outputs": [],
      "source": [
        "param_test2 = {\n",
        " 'gamma':[i/10.0 for i in range(0,5)]\n",
        "}\n",
        "gsearch2 = GridSearchCV(estimator = XGBClassifier( random_state=42,learning_rate =0.1, n_estimators=140, max_depth=9,\n",
        " min_child_weight=6, gamma=0, subsample=0.8, colsample_bytree=0.8,\n",
        " objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), \n",
        " param_grid = param_test2,n_jobs=4,cv=5,verbose=3)"
      ],
      "id": "80JDvFHzEIWL"
    },
    {
      "cell_type": "code",
      "execution_count": 268,
      "metadata": {
        "id": "VkTHma5OEpdL"
      },
      "outputs": [],
      "source": [
        "xgb_tuning2 = Pipeline([('vect', cv),\n",
        "                     ('tfidf', TfidfTransformer()),\n",
        "                    ('gridsearch', gsearch2),\n",
        "])"
      ],
      "id": "VkTHma5OEpdL"
    },
    {
      "cell_type": "code",
      "execution_count": 269,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-OeSDB58ErQq",
        "outputId": "26d34ea1-ea98-40e3-a6e6-8c88e7bfb02f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 5 folds for each of 5 candidates, totalling 25 fits\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(steps=[('vect',\n",
              "                 CountVectorizer(tokenizer=<bound method RegexpTokenizer.tokenize of RegexpTokenizer(pattern='[a-zA-Z0-9]+', gaps=False, discard_empty=True, flags=<RegexFlag.UNICODE|DOTALL|MULTILINE: 56>)>)),\n",
              "                ('tfidf', TfidfTransformer()),\n",
              "                ('gridsearch',\n",
              "                 GridSearchCV(cv=5,\n",
              "                              estimator=XGBClassifier(colsample_bytree=0.8,\n",
              "                                                      max_depth=9,\n",
              "                                                      min_child_weight=6,\n",
              "                                                      n_estimators=140,\n",
              "                                                      nthread=4,\n",
              "                                                      random_state=42, seed=27,\n",
              "                                                      subsample=0.8),\n",
              "                              n_jobs=4,\n",
              "                              param_grid={'gamma': [0.0, 0.1, 0.2, 0.3, 0.4]},\n",
              "                              verbose=3))])"
            ]
          },
          "metadata": {},
          "execution_count": 269
        }
      ],
      "source": [
        "xgb_tuning2.fit(x_train,y_train)"
      ],
      "id": "-OeSDB58ErQq"
    },
    {
      "cell_type": "code",
      "execution_count": 270,
      "metadata": {
        "id": "luKNQv_eE2YX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "77260a1e-1860-4f47-ea22-51386b4b5653"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'gamma': 0.2}\n",
            "XGBClassifier(colsample_bytree=0.8, gamma=0.2, max_depth=9, min_child_weight=6,\n",
            "              n_estimators=140, nthread=4, random_state=42, seed=27,\n",
            "              subsample=0.8)\n"
          ]
        }
      ],
      "source": [
        "# best parameter after tuning\n",
        "print(gsearch2.best_params_)\n",
        " \n",
        "# Model terbaik setelah hyper-parameter tuning\n",
        "print(gsearch2.best_estimator_)"
      ],
      "id": "luKNQv_eE2YX"
    },
    {
      "cell_type": "code",
      "execution_count": 271,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v9S2IVLHDfa4",
        "outputId": "28f9e065-8d1a-48a4-992f-7f101e789ecc"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(steps=[('vect',\n",
              "                 CountVectorizer(tokenizer=<bound method RegexpTokenizer.tokenize of RegexpTokenizer(pattern='[a-zA-Z0-9]+', gaps=False, discard_empty=True, flags=<RegexFlag.UNICODE|DOTALL|MULTILINE: 56>)>)),\n",
              "                ('tfidf', TfidfTransformer()),\n",
              "                ('clf',\n",
              "                 XGBClassifier(colsample_bytree=0.8, gamma=0.2, max_depth=9,\n",
              "                               n_estimators=1000, nthread=4, random_state=42,\n",
              "                               seed=27, subsample=0.8))])"
            ]
          },
          "metadata": {},
          "execution_count": 271
        }
      ],
      "source": [
        "xgb_tuned = XGBClassifier(random_state=42,\n",
        "                     learning_rate =0.1,\n",
        " n_estimators=1000,\n",
        " max_depth=9,\n",
        " min_child_weight=1,\n",
        " gamma=0.2,\n",
        " subsample=0.8,\n",
        " colsample_bytree=0.8,\n",
        " objective= 'binary:logistic',\n",
        " nthread=4,\n",
        " scale_pos_weight=1,\n",
        " seed=27)\n",
        "xgb_clf_tuned = Pipeline([('vect', cv),\n",
        "                     ('tfidf', TfidfTransformer()),\n",
        "                    ('clf', xgb_tuned),\n",
        "])\n",
        "# Train the model using training sets\n",
        "xgb_clf_tuned.fit(x_train, y_train)"
      ],
      "id": "v9S2IVLHDfa4"
    },
    {
      "cell_type": "code",
      "execution_count": 272,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pIKuqmG3DjXb",
        "outputId": "f9c35db2-aaa9-4ee6-dcf3-5dd9d7fb8262"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy\t: 0.8446428571428571\n",
            "F1\t\t: 0.6544876886417793\n",
            "Confusion Matrix:\n",
            " [[1953  295]\n",
            " [ 140  412]]\n",
            "Classification Report:\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "          no       0.93      0.87      0.90      2248\n",
            "         yes       0.58      0.75      0.65       552\n",
            "\n",
            "    accuracy                           0.84      2800\n",
            "   macro avg       0.76      0.81      0.78      2800\n",
            "weighted avg       0.86      0.84      0.85      2800\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Predict test\n",
        "xgb_pred = xgb_clf_tuned.predict(x_test)\n",
        "evaluation(xgb_pred,y_test)"
      ],
      "id": "pIKuqmG3DjXb"
    },
    {
      "cell_type": "code",
      "source": [
        "# Predict val\n",
        "xgb_pred_val = xgb_clf_tuned.predict(x_val)\n",
        "evaluation(xgb_pred_val,y_val)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iZECbGJvdxDM",
        "outputId": "a12fe136-fe18-404e-b7f1-4c8671446540"
      },
      "id": "iZECbGJvdxDM",
      "execution_count": 277,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy\t: 0.8446428571428571\n",
            "F1\t\t: 0.6544876886417793\n",
            "Confusion Matrix:\n",
            " [[1953  295]\n",
            " [ 140  412]]\n",
            "Classification Report:\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "          no       0.93      0.87      0.90      2248\n",
            "         yes       0.58      0.75      0.65       552\n",
            "\n",
            "    accuracy                           0.84      2800\n",
            "   macro avg       0.76      0.81      0.78      2800\n",
            "weighted avg       0.86      0.84      0.85      2800\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "99352246"
      },
      "source": [
        "# Referensi\n",
        "- Stopwords: https://github.com/masdevid/ID-Stopwords\n",
        "- Kamus alay: https://raw.githubusercontent.com/okkyibrohim/id-multi-label-hate-speech-and-abusive-language-detection/master/new_kamusalay.csv"
      ],
      "id": "99352246"
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "toc_visible": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.7"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}