Machine_Learning_Final.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9RV17S3OOkDX"
      },
      "source": [
        "# Final Project - Machine Learning\n",
        "###  Andrew Alvarez"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2u68cFzjiD-1"
      },
      "source": [
        "## Load Packages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VgyDXjHYpTnL",
        "outputId": "9a25f23b-ecb2-4e8f-d92c-37392b4795be"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting umap\n",
            "  Downloading umap-0.1.1.tar.gz (3.2 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Building wheels for collected packages: umap\n",
            "  Building wheel for umap (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for umap: filename=umap-0.1.1-py3-none-any.whl size=3542 sha256=6c8bff8f841bc07a8088cc6de9c8f2f40ce62e775d66bb81d21e31fa785f650b\n",
            "  Stored in directory: /root/.cache/pip/wheels/15/f1/28/53dcf7a309118ed35d810a5f9cb995217800f3f269ab5771cb\n",
            "Successfully built umap\n",
            "Installing collected packages: umap\n",
            "Successfully installed umap-0.1.1\n"
          ]
        }
      ],
      "source": [
        "pip install umap"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2I72UvzJOkDZ",
        "outputId": "bb5f664d-fe7b-4951-8705-284b37e38184"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting bertopic\n",
            "  Downloading bertopic-0.14.1-py2.py3-none-any.whl (120 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m120.7/120.7 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.20.0 in /usr/local/lib/python3.10/dist-packages (from bertopic) (1.22.4)\n",
            "Collecting hdbscan>=0.8.29 (from bertopic)\n",
            "  Downloading hdbscan-0.8.29.tar.gz (5.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.2/5.2 MB\u001b[0m \u001b[31m95.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting umap-learn>=0.5.0 (from bertopic)\n",
            "  Downloading umap-learn-0.5.3.tar.gz (88 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m88.2/88.2 kB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: pandas>=1.1.5 in /usr/local/lib/python3.10/dist-packages (from bertopic) (1.5.3)\n",
            "Requirement already satisfied: scikit-learn>=0.22.2.post1 in /usr/local/lib/python3.10/dist-packages (from bertopic) (1.2.2)\n",
            "Requirement already satisfied: tqdm>=4.41.1 in /usr/local/lib/python3.10/dist-packages (from bertopic) (4.65.0)\n",
            "Collecting sentence-transformers>=0.4.1 (from bertopic)\n",
            "  Downloading sentence-transformers-2.2.2.tar.gz (85 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.0/86.0 kB\u001b[0m \u001b[31m10.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: plotly>=4.7.0 in /usr/local/lib/python3.10/dist-packages (from bertopic) (5.13.1)\n",
            "Requirement already satisfied: cython>=0.27 in /usr/local/lib/python3.10/dist-packages (from hdbscan>=0.8.29->bertopic) (0.29.34)\n",
            "Requirement already satisfied: scipy>=1.0 in /usr/local/lib/python3.10/dist-packages (from hdbscan>=0.8.29->bertopic) (1.10.1)\n",
            "Requirement already satisfied: joblib>=1.0 in /usr/local/lib/python3.10/dist-packages (from hdbscan>=0.8.29->bertopic) (1.2.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.1.5->bertopic) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.1.5->bertopic) (2022.7.1)\n",
            "Requirement already satisfied: tenacity>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from plotly>=4.7.0->bertopic) (8.2.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.22.2.post1->bertopic) (3.1.0)\n",
            "Collecting transformers<5.0.0,>=4.6.0 (from sentence-transformers>=0.4.1->bertopic)\n",
            "  Downloading transformers-4.29.2-py3-none-any.whl (7.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.1/7.1 MB\u001b[0m \u001b[31m91.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=0.4.1->bertopic) (2.0.1+cu118)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=0.4.1->bertopic) (0.15.2+cu118)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=0.4.1->bertopic) (3.8.1)\n",
            "Collecting sentencepiece (from sentence-transformers>=0.4.1->bertopic)\n",
            "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m48.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting huggingface-hub>=0.4.0 (from sentence-transformers>=0.4.1->bertopic)\n",
            "  Downloading huggingface_hub-0.14.1-py3-none-any.whl (224 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m27.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numba>=0.49 in /usr/local/lib/python3.10/dist-packages (from umap-learn>=0.5.0->bertopic) (0.56.4)\n",
            "Collecting pynndescent>=0.5 (from umap-learn>=0.5.0->bertopic)\n",
            "  Downloading pynndescent-0.5.10.tar.gz (1.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m77.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (3.12.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (2023.4.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (2.27.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (4.5.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (23.1)\n",
            "Requirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.49->umap-learn>=0.5.0->bertopic) (0.39.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from numba>=0.49->umap-learn>=0.5.0->bertopic) (67.7.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas>=1.1.5->bertopic) (1.16.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=0.4.1->bertopic) (1.11.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=0.4.1->bertopic) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=0.4.1->bertopic) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=0.4.1->bertopic) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->sentence-transformers>=0.4.1->bertopic) (3.25.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->sentence-transformers>=0.4.1->bertopic) (16.0.5)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers>=0.4.1->bertopic) (2022.10.31)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers<5.0.0,>=4.6.0->sentence-transformers>=0.4.1->bertopic)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m104.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->sentence-transformers>=0.4.1->bertopic) (8.1.3)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision->sentence-transformers>=0.4.1->bertopic) (8.4.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.6.0->sentence-transformers>=0.4.1->bertopic) (2.1.2)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (3.4)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.6.0->sentence-transformers>=0.4.1->bertopic) (1.3.0)\n",
            "Building wheels for collected packages: hdbscan, sentence-transformers, umap-learn, pynndescent\n",
            "  Building wheel for hdbscan (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for hdbscan: filename=hdbscan-0.8.29-cp310-cp310-linux_x86_64.whl size=3541408 sha256=7b2ff1ec2effc00b82a631e631e5b9b266fb19f8148ae6137f34be639a18c821\n",
            "  Stored in directory: /root/.cache/pip/wheels/dc/52/e3/6c6b60b126b4d5c4370cb5ac071b82950f91649d62d72f7f56\n",
            "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sentence-transformers: filename=sentence_transformers-2.2.2-py3-none-any.whl size=125926 sha256=217f20b1c9b90f8d3859c24e70308857136be100640ec9265ddecf7773cfc3eb\n",
            "  Stored in directory: /root/.cache/pip/wheels/62/f2/10/1e606fd5f02395388f74e7462910fe851042f97238cbbd902f\n",
            "  Building wheel for umap-learn (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for umap-learn: filename=umap_learn-0.5.3-py3-none-any.whl size=82816 sha256=7c0db584c52444e19f88fb67a8b800468664e6352d34c113ebbc61fd80e7bb44\n",
            "  Stored in directory: /root/.cache/pip/wheels/a0/e8/c6/a37ea663620bd5200ea1ba0907ab3c217042c1d035ef606acc\n",
            "  Building wheel for pynndescent (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pynndescent: filename=pynndescent-0.5.10-py3-none-any.whl size=55622 sha256=ec04344e02ef35c190e204974849b425416c4f8f70081432ce3b00277d12b988\n",
            "  Stored in directory: /root/.cache/pip/wheels/4a/38/5d/f60a40a66a9512b7e5e83517ebc2d1b42d857be97d135f1096\n",
            "Successfully built hdbscan sentence-transformers umap-learn pynndescent\n",
            "Installing collected packages: tokenizers, sentencepiece, huggingface-hub, transformers, pynndescent, hdbscan, umap-learn, sentence-transformers, bertopic\n",
            "Successfully installed bertopic-0.14.1 hdbscan-0.8.29 huggingface-hub-0.14.1 pynndescent-0.5.10 sentence-transformers-2.2.2 sentencepiece-0.1.99 tokenizers-0.13.3 transformers-4.29.2 umap-learn-0.5.3\n"
          ]
        }
      ],
      "source": [
        "pip install bertopic"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dexj5yZeOkDa",
        "outputId": "06d650c7-a4dc-4b0f-bf9b-8276c9e107fb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: gensim in /usr/local/lib/python3.10/dist-packages (4.3.1)\n",
            "Requirement already satisfied: numpy>=1.18.5 in /usr/local/lib/python3.10/dist-packages (from gensim) (1.22.4)\n",
            "Requirement already satisfied: scipy>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from gensim) (1.10.1)\n",
            "Requirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.10/dist-packages (from gensim) (6.3.0)\n"
          ]
        }
      ],
      "source": [
        "pip install gensim"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3LQJ3LlUOkDc",
        "outputId": "b243cae3-806b-4bf5-ec96-754784e4dfe1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting clean-text\n",
            "  Downloading clean_text-0.6.0-py3-none-any.whl (11 kB)\n",
            "Collecting emoji<2.0.0,>=1.0.0 (from clean-text)\n",
            "  Downloading emoji-1.7.0.tar.gz (175 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m175.4/175.4 kB\u001b[0m \u001b[31m10.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting ftfy<7.0,>=6.0 (from clean-text)\n",
            "  Downloading ftfy-6.1.1-py3-none-any.whl (53 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.1/53.1 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: wcwidth>=0.2.5 in /usr/local/lib/python3.10/dist-packages (from ftfy<7.0,>=6.0->clean-text) (0.2.6)\n",
            "Building wheels for collected packages: emoji\n",
            "  Building wheel for emoji (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for emoji: filename=emoji-1.7.0-py3-none-any.whl size=171033 sha256=0c3567fb9934019ad8da7868cdbe3da18bc5aa3d7117cafc0fc661fb79011126\n",
            "  Stored in directory: /root/.cache/pip/wheels/31/8a/8c/315c9e5d7773f74b33d5ed33f075b49c6eaeb7cedbb86e2cf8\n",
            "Successfully built emoji\n",
            "Installing collected packages: emoji, ftfy, clean-text\n",
            "Successfully installed clean-text-0.6.0 emoji-1.7.0 ftfy-6.1.1\n"
          ]
        }
      ],
      "source": [
        "!pip install clean-text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "xIeEnVkyOkDa"
      },
      "outputs": [],
      "source": [
        "from bertopic import BERTopic\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import nltk as nltk\n",
        "from nltk.corpus import stopwords\n",
        "import gensim\n",
        "from gensim import corpora, models\n",
        "from gensim.models.ldamulticore import LdaMulticore\n",
        "from gensim.utils import simple_preprocess\n",
        "from gensim.models import CoherenceModel\n",
        "import re\n",
        "pd.set_option('display.max_colwidth', -1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8DEywBjIOkDa"
      },
      "source": [
        "## Load data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "tnLw00Kv8jnX"
      },
      "outputs": [],
      "source": [
        "import locale\n",
        "locale.getpreferredencoding = lambda: \"UTF-8\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qlAe4eOkPvKR",
        "outputId": "ebe23500-4a8b-4300-f5db-7badb9bee2ae"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n",
            "/content\n",
            "/content/gdrive/My Drive/Colab Notebooks/Machine Learning/Final\n",
            "'Machine Learning Final.ipynb'\t\t'Machine Learning Final_stripped.ipynb'\n",
            "'Machine Learning Final_stripped.html'\t tweets_clean.xlsx\n",
            "Working Directory\n",
            "/content/gdrive/My Drive/Colab Notebooks/Machine Learning/Final\n"
          ]
        }
      ],
      "source": [
        "#S2 Mount Google Drive to Colab Enviorment\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "!pwd\n",
        "\n",
        "#S3 Establish working directory\n",
        "import os # Operation System\n",
        "os.getcwd()\n",
        "%cd /content/gdrive/My Drive/Colab Notebooks/Machine Learning/Final\n",
        "\n",
        "\n",
        "\n",
        "!ls\n",
        "print('Working Directory')\n",
        "print(os.getcwd())\n",
        "work_dir = \"/content/gdrive/My Drive/Colab Notebooks/Machine Learning/Final\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "TnJNhQyPOkDb"
      },
      "outputs": [],
      "source": [
        "tweets = pd.read_excel('tweets_clean.xlsx') \n",
        "\n",
        "#final_project = pd.read_csv('console_edit.csv') \n",
        "#final_project = pd.read_csv('/Users/andrewalvarez/Dropbox/Mac (2)/Desktop/Twitter/console_edit.csv') "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ta51PPEmOkDc"
      },
      "source": [
        "## Articles clean-up, filtering, EDA"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F5lk7NN-OkDc",
        "outputId": "6df92f30-f0bb-47cf-e2eb-dc4f6d06a6de"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:root:Since the GPL-licensed package `unidecode` is not installed, using Python's `unicodedata` package which yields worse results.\n"
          ]
        }
      ],
      "source": [
        "from cleantext import clean\n",
        "# Removing duplicate text and title\n",
        "tweets = tweets.drop_duplicates(subset='text')\n",
        "\n",
        "# Clean Data for special characters and numbers\n",
        "tweets['text'] = tweets['text'] .apply(lambda x: re.sub('[^A-Za-z.]+', ' ', x.lower()))\n",
        "tweets['text']  = tweets['text'] .apply(lambda x: clean(x, no_emoji=True))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "gRsThSt2OkDd",
        "outputId": "5c690b16-1fa0-4673-9e76-0c7aa6b7d316"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   tweet_id airline            name  \\\n",
              "0  1         United  rdowning76       \n",
              "1  2         United  CoreyAStewart    \n",
              "2  3         United  CoralReefer420   \n",
              "3  4         United  lsalazarll       \n",
              "4  5         United  rombaa           \n",
              "\n",
              "                                                                                                                                       text  \\\n",
              "0  united thanks                                                                                                                              \n",
              "1  united thanks for taking care of that mr happy customer.                                                                                   \n",
              "2  united still no refund or word via dm. please resolve this issue as your cancelled flightled flight was useless to my assistant s trip.    \n",
              "3  united delayed due to lack of crew and now delayed again because there s a long line for deicing... still need to improve service united   \n",
              "4  united thanks we filled it out. how s our luck with this is it common                                                                      \n",
              "\n",
              "        tweet_created  \n",
              "0 2015-02-24 11:42:48  \n",
              "1 2015-02-24 11:42:41  \n",
              "2 2015-02-24 11:39:45  \n",
              "3 2015-02-24 11:39:25  \n",
              "4 2015-02-24 11:38:15  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8dde9b5d-7d1c-4f11-9220-21962a1b5aa6\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>airline</th>\n",
              "      <th>name</th>\n",
              "      <th>text</th>\n",
              "      <th>tweet_created</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>United</td>\n",
              "      <td>rdowning76</td>\n",
              "      <td>united thanks</td>\n",
              "      <td>2015-02-24 11:42:48</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>United</td>\n",
              "      <td>CoreyAStewart</td>\n",
              "      <td>united thanks for taking care of that mr happy customer.</td>\n",
              "      <td>2015-02-24 11:42:41</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>United</td>\n",
              "      <td>CoralReefer420</td>\n",
              "      <td>united still no refund or word via dm. please resolve this issue as your cancelled flightled flight was useless to my assistant s trip.</td>\n",
              "      <td>2015-02-24 11:39:45</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>United</td>\n",
              "      <td>lsalazarll</td>\n",
              "      <td>united delayed due to lack of crew and now delayed again because there s a long line for deicing... still need to improve service united</td>\n",
              "      <td>2015-02-24 11:39:25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>United</td>\n",
              "      <td>rombaa</td>\n",
              "      <td>united thanks we filled it out. how s our luck with this is it common</td>\n",
              "      <td>2015-02-24 11:38:15</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8dde9b5d-7d1c-4f11-9220-21962a1b5aa6')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8dde9b5d-7d1c-4f11-9220-21962a1b5aa6 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8dde9b5d-7d1c-4f11-9220-21962a1b5aa6');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "tweets.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rk25sp7MWh-i",
        "outputId": "4b07e94b-9bb7-4d70-cb51-1f9aaa6a115e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 8807 entries, 0 to 9000\n",
            "Data columns (total 5 columns):\n",
            " #   Column         Non-Null Count  Dtype         \n",
            "---  ------         --------------  -----         \n",
            " 0   tweet_id       8807 non-null   int64         \n",
            " 1   airline        8807 non-null   object        \n",
            " 2   name           8807 non-null   object        \n",
            " 3   text           8807 non-null   object        \n",
            " 4   tweet_created  8807 non-null   datetime64[ns]\n",
            "dtypes: datetime64[ns](1), int64(1), object(3)\n",
            "memory usage: 412.8+ KB\n"
          ]
        }
      ],
      "source": [
        "tweets.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "77H79bCQN618",
        "outputId": "899a9247-8d1d-47c6-df2c-32f865a30ca0"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "United       0.432043\n",
              "American     0.294538\n",
              "Southwest    0.273419\n",
              "Name: airline, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ],
      "source": [
        "#distribution of tweets by airline\n",
        "tweets.airline.value_counts(normalize = True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LxXE-OOvVc8l"
      },
      "source": [
        "Good with this distribution of tweets, happy our client has the highest volume to analyze but not too skewed.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ziFNS7sYosdy"
      },
      "source": [
        "## Sentiment Analysis on All Airlines\n",
        "\n",
        "Run sentiment analysis to understand which airlines are considered most positive and negative at a high level."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 180,
          "referenced_widgets": [
            "e73cf5f568e94ccb9ea5b8b183b5a649",
            "93c3e699b455451e852fba9dbbfc8281",
            "8eeb0b3b48af466eab828a6e8fef0e31",
            "29bb6c3d6408417698e018c8cb9a9eb7",
            "8850a4f6b3aa41c39099a6d02f79b96c",
            "c2f6e48e68b84133b9166d2dd5a863b4",
            "482caf7196be4e9ea25a9778f21b5784",
            "13b80df8601b4c6d8b3f00335a37351a",
            "7503d7d83eaf4e5785fd6921f337e685",
            "879d3fc0b539414485c40deca6e84b26",
            "613a24453e4e4c76b279b54ed6efc6d6",
            "9a435a9366f042daaccf7c7750aa0df9",
            "5e89c82af1cb4adcb7862070f5ca32a9",
            "227e6fcd0fa747f5866a8a84a0f7d939",
            "45ca48b1635341a6bd841e4086849a83",
            "59df55a999b14f6c935435f59ee22e14",
            "66deeb09ca17424ebe4ff2ed13e7c4d4",
            "3d7b5e411cb545a084808bfa45586c53",
            "08533a987f194c1988f116579c020702",
            "944d14bd6a704fbd9958080472bcba04",
            "46cf3bdda26841bcb7d31d8e6ff6e68c",
            "30c0a143a4cf438599e73570fadfd061",
            "406a8e08aea94b6783b22f623eab0ad9",
            "b81056338867489eaad84e735c398c00",
            "5e5f78ead93448b6882e008077f24129",
            "b9f29e8079824b1e96504dd9530c4aed",
            "0df9b72076da4309a18c8ae086c11149",
            "62f21a55f61244c881ba7dc233f0c00e",
            "609ebb90f4a94fa194a9afb1f35e33ab",
            "d5cf96591576405684135e861905c643",
            "a13ff8427cf24bb392186fd3e910c390",
            "86080f75c4cf4d168aa62fcde2cf0843",
            "938de54eebe14716910150f0b4dbb23f",
            "103c42045dd24348b97b9e507716cc59",
            "b2ed16ae612749a89f402c36d3269650",
            "ed7fc4d5f54e4c92a50d06145638b810",
            "0f5838fbf0dd4b2aa23bdbaf4f2e7caa",
            "5fddb72c15da4b428a8da9141b456859",
            "65c28e9f3949455189d8bfddebfa1702",
            "6f944832fbfa40a09777f149c65be8c8",
            "4653a55265f44abfa8b2dc81d6fb7d54",
            "fa9b4f33572642cca0a3156d72ce6d1c",
            "bc4af09af5d74edfa685c452f9612823",
            "e421157b4b1c4e2bb25007cafe1d0efd"
          ]
        },
        "id": "IRjEQBPtQPan",
        "outputId": "7910ed87-da9a-4785-8388-ddb3be033270"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/629 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e73cf5f568e94ccb9ea5b8b183b5a649"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading pytorch_model.bin:   0%|          | 0.00/268M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9a435a9366f042daaccf7c7750aa0df9"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/48.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "406a8e08aea94b6783b22f623eab0ad9"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "103c42045dd24348b97b9e507716cc59"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Xformers is not installed correctly. If you want to use memorry_efficient_attention to accelerate training use the following command to install Xformers\n",
            "pip install xformers.\n"
          ]
        }
      ],
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "sentiment_pipeline = pipeline(\"sentiment-analysis\", model = \"distilbert-base-uncased-finetuned-sst-2-english\", \n",
        "                              framework = \"pt\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6aFpKlwbXF1o"
      },
      "source": [
        "**Pull one tweet and run sentiment pipleine from Hugging Face (https://huggingface.co/transformers/v3.0.2/main_classes/pipelines.html) on it to see if it properly classifies the sentiment before running on full dataset**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "8AzwLMD5wCCI",
        "outputId": "48771a8b-18bd-4ef6-9eb2-e2cd458db918"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      tweet_id   airline             name  \\\n",
              "0     1         United    rdowning76        \n",
              "1     2         United    CoreyAStewart     \n",
              "2     3         United    CoralReefer420    \n",
              "3     4         United    lsalazarll        \n",
              "4     5         United    rombaa            \n",
              "...  ..            ...       ...            \n",
              "8996  8997      American  KristenReenders   \n",
              "8997  8998      American  itsropes          \n",
              "8998  8999      American  sanyabun          \n",
              "8999  9000      American  SraJackson        \n",
              "9000  9001      American  daviddtwu         \n",
              "\n",
              "                                                                                                                                                 text  \\\n",
              "0     united thanks                                                                                                                                     \n",
              "1     united thanks for taking care of that mr happy customer.                                                                                          \n",
              "2     united still no refund or word via dm. please resolve this issue as your cancelled flightled flight was useless to my assistant s trip.           \n",
              "3     united delayed due to lack of crew and now delayed again because there s a long line for deicing... still need to improve service united          \n",
              "4     united thanks we filled it out. how s our luck with this is it common                                                                             \n",
              "...                                                                     ...                                                                             \n",
              "8996  americanair thank you we got on a different flight to chicago.                                                                                    \n",
              "8997  americanair leaving over minutes late flight. no warnings or communication until we were minutes late flight. that s called shitty customer svc   \n",
              "8998  americanair please bring american airlines to blackberry                                                                                          \n",
              "8999  americanair you have my money you change my flight and don t answer your phones any other suggestions so i can make my commitment                 \n",
              "9000  americanair we have ppl so we need know how many seats are on the next flight. plz put us on standby for people on the next flight                \n",
              "\n",
              "           tweet_created  \n",
              "0    2015-02-24 11:42:48  \n",
              "1    2015-02-24 11:42:41  \n",
              "2    2015-02-24 11:39:45  \n",
              "3    2015-02-24 11:39:25  \n",
              "4    2015-02-24 11:38:15  \n",
              "...                  ...  \n",
              "8996 2015-02-22 12:01:01  \n",
              "8997 2015-02-22 11:59:46  \n",
              "8998 2015-02-22 11:59:15  \n",
              "8999 2015-02-22 11:59:02  \n",
              "9000 2015-02-22 11:58:51  \n",
              "\n",
              "[8807 rows x 5 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-37afce43-de79-4866-8cf3-1cc1e9488a47\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>airline</th>\n",
              "      <th>name</th>\n",
              "      <th>text</th>\n",
              "      <th>tweet_created</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>United</td>\n",
              "      <td>rdowning76</td>\n",
              "      <td>united thanks</td>\n",
              "      <td>2015-02-24 11:42:48</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>United</td>\n",
              "      <td>CoreyAStewart</td>\n",
              "      <td>united thanks for taking care of that mr happy customer.</td>\n",
              "      <td>2015-02-24 11:42:41</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>United</td>\n",
              "      <td>CoralReefer420</td>\n",
              "      <td>united still no refund or word via dm. please resolve this issue as your cancelled flightled flight was useless to my assistant s trip.</td>\n",
              "      <td>2015-02-24 11:39:45</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>United</td>\n",
              "      <td>lsalazarll</td>\n",
              "      <td>united delayed due to lack of crew and now delayed again because there s a long line for deicing... still need to improve service united</td>\n",
              "      <td>2015-02-24 11:39:25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>United</td>\n",
              "      <td>rombaa</td>\n",
              "      <td>united thanks we filled it out. how s our luck with this is it common</td>\n",
              "      <td>2015-02-24 11:38:15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8996</th>\n",
              "      <td>8997</td>\n",
              "      <td>American</td>\n",
              "      <td>KristenReenders</td>\n",
              "      <td>americanair thank you we got on a different flight to chicago.</td>\n",
              "      <td>2015-02-22 12:01:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8997</th>\n",
              "      <td>8998</td>\n",
              "      <td>American</td>\n",
              "      <td>itsropes</td>\n",
              "      <td>americanair leaving over minutes late flight. no warnings or communication until we were minutes late flight. that s called shitty customer svc</td>\n",
              "      <td>2015-02-22 11:59:46</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8998</th>\n",
              "      <td>8999</td>\n",
              "      <td>American</td>\n",
              "      <td>sanyabun</td>\n",
              "      <td>americanair please bring american airlines to blackberry</td>\n",
              "      <td>2015-02-22 11:59:15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8999</th>\n",
              "      <td>9000</td>\n",
              "      <td>American</td>\n",
              "      <td>SraJackson</td>\n",
              "      <td>americanair you have my money you change my flight and don t answer your phones any other suggestions so i can make my commitment</td>\n",
              "      <td>2015-02-22 11:59:02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9000</th>\n",
              "      <td>9001</td>\n",
              "      <td>American</td>\n",
              "      <td>daviddtwu</td>\n",
              "      <td>americanair we have ppl so we need know how many seats are on the next flight. plz put us on standby for people on the next flight</td>\n",
              "      <td>2015-02-22 11:58:51</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8807 rows × 5 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-37afce43-de79-4866-8cf3-1cc1e9488a47')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-37afce43-de79-4866-8cf3-1cc1e9488a47 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-37afce43-de79-4866-8cf3-1cc1e9488a47');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ],
      "source": [
        "tweets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "6d8KoJZIQfUO",
        "outputId": "f8e8f799-a618-4463-a54a-d9249c913d46"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'united flying gla mco in a few weeks. how long do we have to be at airport for before departure for both international and domestic ta'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 15
        }
      ],
      "source": [
        "example_tweet = tweets['text'].iloc[10]\n",
        "example_tweet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SnsD1qhXQfIc",
        "outputId": "1455974b-83b7-410e-f1a7-a495c12e4193"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'label': 'NEGATIVE', 'score': 0.9904690384864807}]"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "sentiment_pipeline(example_tweet)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SDB-ZgfftHCj",
        "outputId": "56d49d09-08f2-49ca-9504-3823d949af7c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting openai\n",
            "  Downloading openai-0.27.7-py3-none-any.whl (71 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.0/72.0 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai) (2.27.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai) (4.65.0)\n",
            "Collecting aiohttp (from openai)\n",
            "  Downloading aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m46.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai) (3.4)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (23.1.0)\n",
            "Collecting multidict<7.0,>=4.5 (from aiohttp->openai)\n",
            "  Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3 (from aiohttp->openai)\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting yarl<2.0,>=1.0 (from aiohttp->openai)\n",
            "  Downloading yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting frozenlist>=1.1.1 (from aiohttp->openai)\n",
            "  Downloading frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (149 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiosignal>=1.1.2 (from aiohttp->openai)\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Installing collected packages: multidict, frozenlist, async-timeout, yarl, aiosignal, aiohttp, openai\n",
            "Successfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 frozenlist-1.3.3 multidict-6.0.4 openai-0.27.7 yarl-1.9.2\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting python-dotenv\n",
            "  Downloading python_dotenv-1.0.0-py3-none-any.whl (19 kB)\n",
            "Installing collected packages: python-dotenv\n",
            "Successfully installed python-dotenv-1.0.0\n"
          ]
        }
      ],
      "source": [
        "!pip install openai\n",
        "!pip install python-dotenv"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "GV93-0Ojl7X3"
      },
      "outputs": [],
      "source": [
        "import openai\n",
        "import os\n",
        "\n",
        "from dotenv import load_dotenv, find_dotenv\n",
        "_ = load_dotenv(find_dotenv()) # read local .env file\n",
        "\n",
        "openai.api_key  = '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'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "VPhVdkeul8Nh"
      },
      "outputs": [],
      "source": [
        "def get_completion(prompt, model=\"gpt-3.5-turbo\"):\n",
        "    messages = [{\"role\": \"user\", \"content\": prompt}]\n",
        "    response = openai.ChatCompletion.create(\n",
        "        model=model,\n",
        "        messages=messages,\n",
        "        temperature=0, # this is the degree of randomness of the model's output\n",
        "    )\n",
        "    return response.choices[0].message[\"content\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "BqrziWKq0PZ-"
      },
      "outputs": [],
      "source": [
        "# question = \"3\"\n",
        "# prompt = \"\"\"Categorize the sentiment of the writer as POSITIVE or NEGATIVE based on the text wrapped by triple backtick\n",
        "# return the result as a json value with the key name as label for value of  POSITIVE or NEGATIVE only\n",
        "# you will return right after \"You will return\"\n",
        "\n",
        "# Question:\n",
        "# ```{question}```\n",
        "# You will return\n",
        "# \"\"\"\n",
        "# response = get_completion(prompt.format(question = question))\n",
        "# print(response)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "nd_vg_Uhl7RR"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "def sentiment_checker(question):\n",
        "  prompt = \"\"\"Categorize the sentiment of the writer as POSITIVE or NEGATIVE based on the text wrapped by triple backtick\n",
        "  return the result as a json value with the key name as label for value of  POSITIVE or NEGATIVE only\n",
        "  you will return right after \"You will return\"\n",
        "\n",
        "  Question:\n",
        "  ```{question}```\n",
        "  You will return\n",
        "  \"\"\"\n",
        "\n",
        "  response = get_completion(prompt.format(question = question))\n",
        "  return json.loads(response)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "lkkI_nYGl7Kk"
      },
      "outputs": [],
      "source": [
        "\n",
        "# import random\n",
        "# import time\n",
        "\n",
        "# total_sample = 900\n",
        "# correct_sample = 0 #it's a good habit to have it defined as 0 as a default value and go from there\n",
        "# df = pd.DataFrame(columns=['model_label', 'gpt_label'])\n",
        "# #current_step relates to enumerate. It will return a paired array of index and element (current_step is index and value of element is the i, which is the randomized number)\n",
        "# #square bracket usually means list or indexing\n",
        "# for current_step, i in enumerate([ random.randint(0, tweets.shape[0]) for _ in range(total_sample) ]):\n",
        "#   #print(current_step)\n",
        "#   tweet = tweets['text'].iloc[i]\n",
        "#   model_response = sentiment_pipeline(tweet)[0]\n",
        "#   gpt_response = sentiment_checker(tweet)\n",
        "#   if model_response['label'] == gpt_response['label']: correct_sample += 1\n",
        "#   new_data = { 'model_label': model_response['label'], 'gpt_label' : gpt_response['label']}\n",
        "#   df = df.append(new_data, ignore_index=True)\n",
        "#   time.sleep(5)\n",
        "\n",
        "# print('total samples: {}, correct samples: {}, accuracy: {}'.format(total_sample, correct_sample, correct_sample / total_sample))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "dmyZC990TgCf"
      },
      "outputs": [],
      "source": [
        "# import random\n",
        "# import time\n",
        "# # import pandas as pd  \n",
        "# from json.decoder import JSONDecodeError\n",
        "\n",
        "# total_sample = 500\n",
        "# correct_sample = 0\n",
        "# df = pd.DataFrame(columns=['model_label', 'gpt_label'])\n",
        "\n",
        "# for current_step, i in enumerate([random.randint(0, tweets.shape[0]) for _ in range(total_sample)]):\n",
        "#     tweet = tweets['text'].iloc[i]\n",
        "#     try:\n",
        "#         model_response = sentiment_pipeline(tweet)[0]\n",
        "#         gpt_response = sentiment_checker(tweet)\n",
        "#     except JSONDecodeError as e:\n",
        "#         print(f\"Error occurred at tweet index {i} with content: {tweet}\")\n",
        "#         print(f\"Error details: {str(e)}\")\n",
        "#         continue\n",
        "\n",
        "#     if model_response['label'] == gpt_response['label']:\n",
        "#         correct_sample += 1\n",
        "\n",
        "#     new_data = {'model_label': model_response['label'], 'gpt_label': gpt_response['label']}\n",
        "#     df = df.append(new_data, ignore_index=True)\n",
        "#     time.sleep(20)\n",
        "\n",
        "# print('total samples: {}, correct samples: {}, accuracy: {}'.format(total_sample, correct_sample, correct_sample / total_sample))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "T1zL5jHU1yUu"
      },
      "outputs": [],
      "source": [
        "# import random\n",
        "# import time\n",
        "# # import pandas as pd  \n",
        "# from json.decoder import JSONDecodeError\n",
        "\n",
        "# total_sample = 2000\n",
        "# correct_sample = 0\n",
        "# df = pd.DataFrame(columns=['model_label', 'gpt_label'])\n",
        "# mismatched_df = pd.DataFrame(columns=['text', 'model_label', 'gpt_label'])\n",
        "# match_df = pd.DataFrame(columns=['text', 'model_label', 'gpt_label'])\n",
        "\n",
        "# for current_step, i in enumerate([random.randint(0, tweets.shape[0] - 1) for _ in range(total_sample)]): # -1 to prevent an out of bounds error\n",
        "#     tweet = tweets['text'].iloc[i]\n",
        "#     try:\n",
        "#         model_response = sentiment_pipeline(tweet)[0]\n",
        "#         gpt_response = sentiment_checker(tweet)\n",
        "#     except JSONDecodeError as e:\n",
        "#         print(f\"Error occurred at tweet index {i} with content: {tweet}\")\n",
        "#         print(f\"Error details: {str(e)}\")\n",
        "#         continue\n",
        "#     except openai.error.RateLimitError as e:  # Handling RateLimitError from the openai library\n",
        "#         print(f\"Rate limit error occurred at tweet index {i} with content: {tweet}\")\n",
        "#         print(f\"Error details: {str(e)}\")\n",
        "#         time.sleep(90)  # Wait for 180 seconds before retrying\n",
        "#         continue\n",
        "\n",
        "#     if model_response['label'] == gpt_response['label']:\n",
        "#         correct_sample += 1\n",
        "#         match_data = { 'text': tweet, 'model_label': model_response['label'], 'gpt_label': gpt_response['label']}\n",
        "#         match_df = match_df.append(match_data, ignore_index=True)\n",
        "#     else:\n",
        "#         new_mismatch_data = { 'text': tweet, 'model_label': model_response['label'], 'gpt_label': gpt_response['label']}\n",
        "#         mismatched_df = mismatched_df.append(new_mismatch_data, ignore_index=True)\n",
        "\n",
        "#     new_data = {'model_label': model_response['label'], 'gpt_label': gpt_response['label']}\n",
        "#     df = df.append(new_data, ignore_index=True)\n",
        "#     # time.sleep(10)\n",
        "\n",
        "# print('total samples: {}, correct samples: {}, accuracy: {}'.format(total_sample, correct_sample, correct_sample / total_sample))\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "import time\n",
        "# import openai\n",
        "# import pandas as pd  \n",
        "from json.decoder import JSONDecodeError\n",
        "\n",
        "total_sample = 2000\n",
        "correct_sample = 0\n",
        "df = pd.DataFrame(columns=['model_label', 'gpt_label'])\n",
        "mismatched_df = pd.DataFrame(columns=['text', 'model_label', 'gpt_label'])\n",
        "match_df = pd.DataFrame(columns=['text', 'model_label', 'gpt_label'])\n",
        "\n",
        "for current_step, i in enumerate([random.randint(0, tweets.shape[0] - 1) for _ in range(total_sample)]): # -1 to prevent an out of bounds error\n",
        "    tweet = tweets['text'].iloc[i]\n",
        "    try:\n",
        "        model_response = sentiment_pipeline(tweet)[0]\n",
        "        gpt_response = sentiment_checker(tweet)\n",
        "    except JSONDecodeError as e:\n",
        "        # print(f\"Error occurred at tweet index {i} with content: {tweet}\")\n",
        "        # print(f\"Error details: {str(e)}\")\n",
        "        continue\n",
        "    except openai.error.RateLimitError as e:  # Handling RateLimitError from the openai library\n",
        "        # print(f\"Rate limit error occurred at tweet index {i} with content: {tweet}\")\n",
        "        # print(f\"Error details: {str(e)}\")\n",
        "        time.sleep(90)  # Wait for 180 seconds before retrying\n",
        "        continue\n",
        "    except openai.error.OpenAIError as e:  # Handling OpenAIError from the openai library\n",
        "        # print(f\"OpenAI API error occurred at tweet index {i} with content: {tweet}\")\n",
        "        # print(f\"Error details: {str(e)}\")\n",
        "        time.sleep(60)  # Wait for 60 seconds before retrying\n",
        "        continue\n",
        "\n",
        "    if model_response['label'] == gpt_response['label']:\n",
        "        correct_sample += 1\n",
        "        match_data = { 'text': tweet, 'model_label': model_response['label'], 'gpt_label': gpt_response['label']}\n",
        "        match_df = match_df.append(match_data, ignore_index=True)\n",
        "    else:\n",
        "        new_mismatch_data = { 'text': tweet, 'model_label': model_response['label'], 'gpt_label': gpt_response['label']}\n",
        "        mismatched_df = mismatched_df.append(new_mismatch_data, ignore_index=True)\n",
        "\n",
        "    new_data = {'model_label': model_response['label'], 'gpt_label': gpt_response['label']}\n",
        "    df = df.append(new_data, ignore_index=True)\n",
        "    # time.sleep(10)\n",
        "\n",
        "print('total samples: {}, correct samples: {}, accuracy: {}'.format(total_sample, correct_sample, correct_sample / total_sample))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0olPX202-7W5",
        "outputId": "9ed92a2e-8b1c-4d0c-ce97-20d14a48d92e"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "total samples: 2000, correct samples: 1610, accuracy: 0.805\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "L3C8N76VU8BG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "896bf599-21fd-4282-e1d7-f3f6b730482f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "text           americanair thx for showing me that your twitter appreciates me more than your employees. sure another airline would like my nohotel\n",
              "model_label    POSITIVE                                                                                                                            \n",
              "gpt_label      NEGATIVE                                                                                                                            \n",
              "Name: 0, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ],
      "source": [
        "mismatched_df.iloc[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Grm1ROw7QZKM",
        "outputId": "ed3f66e6-2f1f-4676-d507-130de874d77f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "text           united thanks so much my passport was recovered\n",
              "model_label    POSITIVE                                       \n",
              "gpt_label      POSITIVE                                       \n",
              "Name: 0, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ],
      "source": [
        "match_df.iloc[0]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jpy0npJmXg4p"
      },
      "source": [
        "Accuracy of 81% suggests that we can move forward with leveraging DistilBERT **Now let's run it on the entire dataset:**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xQzi7_HxQe-t"
      },
      "outputs": [],
      "source": [
        "sentiment = []\n",
        "for i in tweets['text']:\n",
        "    try:\n",
        "    \n",
        "    \n",
        "        sentiment.append(sentiment_pipeline(i)[0]['label'])\n",
        "    except:\n",
        "        sentiment.append('NEUTRAL')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "algI79TjQey8"
      },
      "outputs": [],
      "source": [
        "tweets['sentiment'] = sentiment"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kcEtp_tpo2ck",
        "outputId": "525b25c2-592a-444e-f98f-9b094de75563"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "NEGATIVE    0.734416\n",
              "POSITIVE    0.265584\n",
              "Name: sentiment, dtype: float64"
            ]
          },
          "execution_count": 27,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "tweets.sentiment.value_counts(normalize = True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 336
        },
        "id": "NZII5lM7pdGO",
        "outputId": "6979493c-7a1d-4dac-c634-dbd8de0d7ce8"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-552ddf7f-84ff-45f1-a8ed-892f4928e428\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>airline</th>\n",
              "      <th>name</th>\n",
              "      <th>text</th>\n",
              "      <th>tweet_created</th>\n",
              "      <th>sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>United</td>\n",
              "      <td>rdowning76</td>\n",
              "      <td>united thanks</td>\n",
              "      <td>2015-02-24 11:42:48</td>\n",
              "      <td>POSITIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>United</td>\n",
              "      <td>CoreyAStewart</td>\n",
              "      <td>united thanks for taking care of that mr happy customer.</td>\n",
              "      <td>2015-02-24 11:42:41</td>\n",
              "      <td>POSITIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>United</td>\n",
              "      <td>CoralReefer420</td>\n",
              "      <td>united still no refund or word via dm. please resolve this issue as your cancelled flightled flight was useless to my assistant s trip.</td>\n",
              "      <td>2015-02-24 11:39:45</td>\n",
              "      <td>NEGATIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>United</td>\n",
              "      <td>lsalazarll</td>\n",
              "      <td>united delayed due to lack of crew and now delayed again because there s a long line for deicing... still need to improve service united</td>\n",
              "      <td>2015-02-24 11:39:25</td>\n",
              "      <td>NEGATIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>United</td>\n",
              "      <td>rombaa</td>\n",
              "      <td>united thanks we filled it out. how s our luck with this is it common</td>\n",
              "      <td>2015-02-24 11:38:15</td>\n",
              "      <td>POSITIVE</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-552ddf7f-84ff-45f1-a8ed-892f4928e428')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-552ddf7f-84ff-45f1-a8ed-892f4928e428 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-552ddf7f-84ff-45f1-a8ed-892f4928e428');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "   tweet_id airline            name  \\\n",
              "0  1         United  rdowning76       \n",
              "1  2         United  CoreyAStewart    \n",
              "2  3         United  CoralReefer420   \n",
              "3  4         United  lsalazarll       \n",
              "4  5         United  rombaa           \n",
              "\n",
              "                                                                                                                                       text  \\\n",
              "0  united thanks                                                                                                                              \n",
              "1  united thanks for taking care of that mr happy customer.                                                                                   \n",
              "2  united still no refund or word via dm. please resolve this issue as your cancelled flightled flight was useless to my assistant s trip.    \n",
              "3  united delayed due to lack of crew and now delayed again because there s a long line for deicing... still need to improve service united   \n",
              "4  united thanks we filled it out. how s our luck with this is it common                                                                      \n",
              "\n",
              "        tweet_created sentiment  \n",
              "0 2015-02-24 11:42:48  POSITIVE  \n",
              "1 2015-02-24 11:42:41  POSITIVE  \n",
              "2 2015-02-24 11:39:45  NEGATIVE  \n",
              "3 2015-02-24 11:39:25  NEGATIVE  \n",
              "4 2015-02-24 11:38:15  POSITIVE  "
            ]
          },
          "execution_count": 28,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "tweets.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 641
        },
        "id": "UB0j8ydGenWH",
        "outputId": "d8fd577b-cf33-4716-cefc-0b34e2cdd9dc"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-754bb3ab-9717-456f-a786-3a0f982642df\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>airline</th>\n",
              "      <th>name</th>\n",
              "      <th>text</th>\n",
              "      <th>tweet_created</th>\n",
              "      <th>sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>United</td>\n",
              "      <td>CoralReefer420</td>\n",
              "      <td>united still no refund or word via dm. please resolve this issue as your cancelled flightled flight was useless to my assistant s trip.</td>\n",
              "      <td>2015-02-24 11:39:45</td>\n",
              "      <td>NEGATIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>United</td>\n",
              "      <td>lsalazarll</td>\n",
              "      <td>united delayed due to lack of crew and now delayed again because there s a long line for deicing... still need to improve service united</td>\n",
              "      <td>2015-02-24 11:39:25</td>\n",
              "      <td>NEGATIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>7</td>\n",
              "      <td>United</td>\n",
              "      <td>karenmcgregor86</td>\n",
              "      <td>united even on international flight glasgow to the u.s. then what about orlando to newark all mins</td>\n",
              "      <td>2015-02-24 11:33:46</td>\n",
              "      <td>NEGATIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>8</td>\n",
              "      <td>United</td>\n",
              "      <td>andycheco</td>\n",
              "      <td>united you think you boarded flight au too early i think so.</td>\n",
              "      <td>2015-02-24 11:31:01</td>\n",
              "      <td>NEGATIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>9</td>\n",
              "      <td>United</td>\n",
              "      <td>hmansfield</td>\n",
              "      <td>united i understand but it s tough when there is no way to get to the airport w o serious risk of an accident. it s a steep price.</td>\n",
              "      <td>2015-02-24 11:20:56</td>\n",
              "      <td>NEGATIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8994</th>\n",
              "      <td>8995</td>\n",
              "      <td>American</td>\n",
              "      <td>RussellsWriting</td>\n",
              "      <td>americanair my flight was cancelled flightled leaving tomorrow morning. auto rebooked for a tuesday night flight but need to arrive monday.</td>\n",
              "      <td>2015-02-22 12:01:06</td>\n",
              "      <td>NEGATIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8995</th>\n",
              "      <td>8996</td>\n",
              "      <td>American</td>\n",
              "      <td>GolfWithWoody</td>\n",
              "      <td>americanair right on cue with the delays</td>\n",
              "      <td>2015-02-22 12:01:02</td>\n",
              "      <td>NEGATIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8997</th>\n",
              "      <td>8998</td>\n",
              "      <td>American</td>\n",
              "      <td>itsropes</td>\n",
              "      <td>americanair leaving over minutes late flight. no warnings or communication until we were minutes late flight. that s called shitty customer svc</td>\n",
              "      <td>2015-02-22 11:59:46</td>\n",
              "      <td>NEGATIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8999</th>\n",
              "      <td>9000</td>\n",
              "      <td>American</td>\n",
              "      <td>SraJackson</td>\n",
              "      <td>americanair you have my money you change my flight and don t answer your phones any other suggestions so i can make my commitment</td>\n",
              "      <td>2015-02-22 11:59:02</td>\n",
              "      <td>NEGATIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9000</th>\n",
              "      <td>9001</td>\n",
              "      <td>American</td>\n",
              "      <td>daviddtwu</td>\n",
              "      <td>americanair we have ppl so we need know how many seats are on the next flight. plz put us on standby for people on the next flight</td>\n",
              "      <td>2015-02-22 11:58:51</td>\n",
              "      <td>NEGATIVE</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>6468 rows × 6 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-754bb3ab-9717-456f-a786-3a0f982642df')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-754bb3ab-9717-456f-a786-3a0f982642df button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-754bb3ab-9717-456f-a786-3a0f982642df');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "      tweet_id   airline             name  \\\n",
              "2     3         United    CoralReefer420    \n",
              "3     4         United    lsalazarll        \n",
              "6     7         United    karenmcgregor86   \n",
              "7     8         United    andycheco         \n",
              "8     9         United    hmansfield        \n",
              "...  ..            ...           ...        \n",
              "8994  8995      American  RussellsWriting   \n",
              "8995  8996      American  GolfWithWoody     \n",
              "8997  8998      American  itsropes          \n",
              "8999  9000      American  SraJackson        \n",
              "9000  9001      American  daviddtwu         \n",
              "\n",
              "                                                                                                                                                 text  \\\n",
              "2     united still no refund or word via dm. please resolve this issue as your cancelled flightled flight was useless to my assistant s trip.           \n",
              "3     united delayed due to lack of crew and now delayed again because there s a long line for deicing... still need to improve service united          \n",
              "6     united even on international flight glasgow to the u.s. then what about orlando to newark all mins                                                \n",
              "7     united you think you boarded flight au too early i think so.                                                                                      \n",
              "8     united i understand but it s tough when there is no way to get to the airport w o serious risk of an accident. it s a steep price.                \n",
              "...                                                                                                                                  ...                \n",
              "8994  americanair my flight was cancelled flightled leaving tomorrow morning. auto rebooked for a tuesday night flight but need to arrive monday.       \n",
              "8995  americanair right on cue with the delays                                                                                                          \n",
              "8997  americanair leaving over minutes late flight. no warnings or communication until we were minutes late flight. that s called shitty customer svc   \n",
              "8999  americanair you have my money you change my flight and don t answer your phones any other suggestions so i can make my commitment                 \n",
              "9000  americanair we have ppl so we need know how many seats are on the next flight. plz put us on standby for people on the next flight                \n",
              "\n",
              "           tweet_created sentiment  \n",
              "2    2015-02-24 11:39:45  NEGATIVE  \n",
              "3    2015-02-24 11:39:25  NEGATIVE  \n",
              "6    2015-02-24 11:33:46  NEGATIVE  \n",
              "7    2015-02-24 11:31:01  NEGATIVE  \n",
              "8    2015-02-24 11:20:56  NEGATIVE  \n",
              "...                  ...       ...  \n",
              "8994 2015-02-22 12:01:06  NEGATIVE  \n",
              "8995 2015-02-22 12:01:02  NEGATIVE  \n",
              "8997 2015-02-22 11:59:46  NEGATIVE  \n",
              "8999 2015-02-22 11:59:02  NEGATIVE  \n",
              "9000 2015-02-22 11:58:51  NEGATIVE  \n",
              "\n",
              "[6468 rows x 6 columns]"
            ]
          },
          "execution_count": 29,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#pull negative tweets to ensure sample looks negative\n",
        "neg = (tweets['sentiment'] == 'NEGATIVE')\n",
        "tweets[neg]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 537
        },
        "id": "ySBva-OVliKH",
        "outputId": "63e00c5c-5c5e-4ef4-9af1-2b188d7038ff"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-5a8ac24d-f912-48d5-8672-519a80b47a4e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>airline</th>\n",
              "      <th>name</th>\n",
              "      <th>text</th>\n",
              "      <th>tweet_created</th>\n",
              "      <th>sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>United</td>\n",
              "      <td>rdowning76</td>\n",
              "      <td>united thanks</td>\n",
              "      <td>2015-02-24 11:42:48</td>\n",
              "      <td>POSITIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>United</td>\n",
              "      <td>CoreyAStewart</td>\n",
              "      <td>united thanks for taking care of that mr happy customer.</td>\n",
              "      <td>2015-02-24 11:42:41</td>\n",
              "      <td>POSITIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>United</td>\n",
              "      <td>rombaa</td>\n",
              "      <td>united thanks we filled it out. how s our luck with this is it common</td>\n",
              "      <td>2015-02-24 11:38:15</td>\n",
              "      <td>POSITIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>6</td>\n",
              "      <td>United</td>\n",
              "      <td>samidip</td>\n",
              "      <td>united your eri ord express connections are hugely popular .. now if only we could have an eri ewr hop</td>\n",
              "      <td>2015-02-24 11:36:12</td>\n",
              "      <td>POSITIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>13</td>\n",
              "      <td>United</td>\n",
              "      <td>BK_TheBri</td>\n",
              "      <td>united thanks. it is on the same ticket.</td>\n",
              "      <td>2015-02-24 11:10:47</td>\n",
              "      <td>POSITIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8988</th>\n",
              "      <td>8989</td>\n",
              "      <td>American</td>\n",
              "      <td>KristinaMeyer7</td>\n",
              "      <td>americanair flight cancelled flightled can t go home until tomorrow. i could use dinner and a play americanair it s my first time in nyc.</td>\n",
              "      <td>2015-02-22 12:05:30</td>\n",
              "      <td>POSITIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8989</th>\n",
              "      <td>8990</td>\n",
              "      <td>American</td>\n",
              "      <td>jlhalldc</td>\n",
              "      <td>thank you. americanair jlhalldc customer relations will review your concerns and contact you back directly john.</td>\n",
              "      <td>2015-02-22 12:04:51</td>\n",
              "      <td>POSITIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8991</th>\n",
              "      <td>8992</td>\n",
              "      <td>American</td>\n",
              "      <td>Laurelinesblog</td>\n",
              "      <td>americanair thanks he is.</td>\n",
              "      <td>2015-02-22 12:04:09</td>\n",
              "      <td>POSITIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8996</th>\n",
              "      <td>8997</td>\n",
              "      <td>American</td>\n",
              "      <td>KristenReenders</td>\n",
              "      <td>americanair thank you we got on a different flight to chicago.</td>\n",
              "      <td>2015-02-22 12:01:01</td>\n",
              "      <td>POSITIVE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8998</th>\n",
              "      <td>8999</td>\n",
              "      <td>American</td>\n",
              "      <td>sanyabun</td>\n",
              "      <td>americanair please bring american airlines to blackberry</td>\n",
              "      <td>2015-02-22 11:59:15</td>\n",
              "      <td>POSITIVE</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2339 rows × 6 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5a8ac24d-f912-48d5-8672-519a80b47a4e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-5a8ac24d-f912-48d5-8672-519a80b47a4e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-5a8ac24d-f912-48d5-8672-519a80b47a4e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "      tweet_id   airline             name  \\\n",
              "0     1         United    rdowning76        \n",
              "1     2         United    CoreyAStewart     \n",
              "4     5         United    rombaa            \n",
              "5     6         United    samidip           \n",
              "12    13        United    BK_TheBri         \n",
              "...   ..           ...          ...         \n",
              "8988  8989      American  KristinaMeyer7    \n",
              "8989  8990      American  jlhalldc          \n",
              "8991  8992      American  Laurelinesblog    \n",
              "8996  8997      American  KristenReenders   \n",
              "8998  8999      American  sanyabun          \n",
              "\n",
              "                                                                                                                                           text  \\\n",
              "0     united thanks                                                                                                                               \n",
              "1     united thanks for taking care of that mr happy customer.                                                                                    \n",
              "4     united thanks we filled it out. how s our luck with this is it common                                                                       \n",
              "5     united your eri ord express connections are hugely popular .. now if only we could have an eri ewr hop                                      \n",
              "12    united thanks. it is on the same ticket.                                                                                                    \n",
              "...                                        ...                                                                                                    \n",
              "8988  americanair flight cancelled flightled can t go home until tomorrow. i could use dinner and a play americanair it s my first time in nyc.   \n",
              "8989  thank you. americanair jlhalldc customer relations will review your concerns and contact you back directly john.                            \n",
              "8991  americanair thanks he is.                                                                                                                   \n",
              "8996  americanair thank you we got on a different flight to chicago.                                                                              \n",
              "8998  americanair please bring american airlines to blackberry                                                                                    \n",
              "\n",
              "           tweet_created sentiment  \n",
              "0    2015-02-24 11:42:48  POSITIVE  \n",
              "1    2015-02-24 11:42:41  POSITIVE  \n",
              "4    2015-02-24 11:38:15  POSITIVE  \n",
              "5    2015-02-24 11:36:12  POSITIVE  \n",
              "12   2015-02-24 11:10:47  POSITIVE  \n",
              "...                  ...       ...  \n",
              "8988 2015-02-22 12:05:30  POSITIVE  \n",
              "8989 2015-02-22 12:04:51  POSITIVE  \n",
              "8991 2015-02-22 12:04:09  POSITIVE  \n",
              "8996 2015-02-22 12:01:01  POSITIVE  \n",
              "8998 2015-02-22 11:59:15  POSITIVE  \n",
              "\n",
              "[2339 rows x 6 columns]"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "\n",
        "pos = (tweets['sentiment'] == 'POSITIVE')\n",
        "tweets[pos]\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DPbtWmdOnWka"
      },
      "source": [
        "Sentiment classification looks good. Moving forward with this."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kj_6L4B5Riq2",
        "outputId": "d4e18153-7638-46a5-d88e-f146a674e8b8"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "airline    sentiment\n",
              "American   NEGATIVE     2010\n",
              "           POSITIVE     584 \n",
              "Southwest  NEGATIVE     1641\n",
              "           POSITIVE     767 \n",
              "United     NEGATIVE     2817\n",
              "           POSITIVE     988 \n",
              "Name: sentiment, dtype: int64"
            ]
          },
          "execution_count": 31,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "airline_sentiments = tweets.groupby(['airline','sentiment'])['sentiment'].count()\n",
        "airline_sentiments"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 268
        },
        "id": "3CJd4A5ipEGp",
        "outputId": "e76add61-2068-45dc-9cb8-0ffe3aa028ea"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-f5d47f3c-f2c7-4ac4-a175-61d1e33159e9\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>sentiment</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>airline</th>\n",
              "      <th>sentiment</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">American</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>77.49</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>22.51</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Southwest</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>68.15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>31.85</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">United</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>74.03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>25.97</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f5d47f3c-f2c7-4ac4-a175-61d1e33159e9')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f5d47f3c-f2c7-4ac4-a175-61d1e33159e9 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f5d47f3c-f2c7-4ac4-a175-61d1e33159e9');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                     sentiment\n",
              "airline   sentiment           \n",
              "American  NEGATIVE   77.49    \n",
              "          POSITIVE   22.51    \n",
              "Southwest NEGATIVE   68.15    \n",
              "          POSITIVE   31.85    \n",
              "United    NEGATIVE   74.03    \n",
              "          POSITIVE   25.97    "
            ]
          },
          "execution_count": 32,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "airline_sentiments = tweets.groupby(['airline', 'sentiment']).agg({'sentiment': 'count'})\n",
        "airlines = tweets.groupby(['airline']).agg({'sentiment': 'count'})\n",
        "sentiment_pctg = round(airline_sentiments.div(airlines, level='airline') * 100,2)\n",
        "sentiment_pctg\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mf8XuDezXL33"
      },
      "source": [
        "**According to the above sentiment analysis, Southwest has the most positive tweet percentage at 32%. United's positive tweet rate is 26% and American's is only 23%**.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eeRgCdt6qsXC"
      },
      "source": [
        "## Topic Modeling -- All Tweets/Airlines\n",
        "\n",
        "Running a topic analysis using BERT package on all of the tweets to understand what are the 10 most important topics according to customers on Twitter"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GyOgzljiVvgt"
      },
      "source": [
        "####  Removing stopwords in tweets from nltk package "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v8ELEk_FOkDe",
        "outputId": "bdffaa27-2c75-4e28-df24-0041199c9a27"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        }
      ],
      "source": [
        "nltk.download('stopwords')\n",
        "data_list = tweets['text'].tolist()        \n",
        "\n",
        "add_stopwords = [\"united\",\"americanair\",\"southwestair\"] #removing airlines so it doesn't topic cluster based on these\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english')))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GsAPCJCWTIz5"
      },
      "source": [
        "#### Stemming --> finding root of each word to enhance interpretability of tweet for model (eg. running --> run)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sCyK5XrbOkDe"
      },
      "outputs": [],
      "source": [
        "from nltk.stem import PorterStemmer \n",
        "ps = PorterStemmer()\n",
        " \n",
        "# choose some words to be stemmed\n",
        "data_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    \n",
        "    for w in i:\n",
        "        \n",
        "        a = ps.stem(w)+\" \"\n",
        "        \n",
        "        string += a\n",
        "    data_clean.append(string)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pyI3vBM54TXB",
        "outputId": "64722364-26ba-4163-e86d-eb6d809d5f96"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: bertopic in /usr/local/lib/python3.10/dist-packages (0.14.1)\n",
            "Requirement already satisfied: numpy>=1.20.0 in /usr/local/lib/python3.10/dist-packages (from bertopic) (1.22.4)\n",
            "Requirement already satisfied: hdbscan>=0.8.29 in /usr/local/lib/python3.10/dist-packages (from bertopic) (0.8.29)\n",
            "Requirement already satisfied: umap-learn>=0.5.0 in /usr/local/lib/python3.10/dist-packages (from bertopic) (0.5.3)\n",
            "Requirement already satisfied: pandas>=1.1.5 in /usr/local/lib/python3.10/dist-packages (from bertopic) (1.5.3)\n",
            "Requirement already satisfied: scikit-learn>=0.22.2.post1 in /usr/local/lib/python3.10/dist-packages (from bertopic) (1.2.2)\n",
            "Requirement already satisfied: tqdm>=4.41.1 in /usr/local/lib/python3.10/dist-packages (from bertopic) (4.65.0)\n",
            "Requirement already satisfied: sentence-transformers>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from bertopic) (2.2.2)\n",
            "Requirement already satisfied: plotly>=4.7.0 in /usr/local/lib/python3.10/dist-packages (from bertopic) (5.13.1)\n",
            "Requirement already satisfied: cython>=0.27 in /usr/local/lib/python3.10/dist-packages (from hdbscan>=0.8.29->bertopic) (0.29.34)\n",
            "Requirement already satisfied: scipy>=1.0 in /usr/local/lib/python3.10/dist-packages (from hdbscan>=0.8.29->bertopic) (1.10.1)\n",
            "Requirement already satisfied: joblib>=1.0 in /usr/local/lib/python3.10/dist-packages (from hdbscan>=0.8.29->bertopic) (1.2.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.1.5->bertopic) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.1.5->bertopic) (2022.7.1)\n",
            "Requirement already satisfied: tenacity>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from plotly>=4.7.0->bertopic) (8.2.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.22.2.post1->bertopic) (3.1.0)\n",
            "Requirement already satisfied: transformers<5.0.0,>=4.6.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=0.4.1->bertopic) (4.29.2)\n",
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=0.4.1->bertopic) (2.0.0+cu118)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=0.4.1->bertopic) (0.15.1+cu118)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=0.4.1->bertopic) (3.8.1)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=0.4.1->bertopic) (0.1.99)\n",
            "Requirement already satisfied: huggingface-hub>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=0.4.1->bertopic) (0.14.1)\n",
            "Requirement already satisfied: numba>=0.49 in /usr/local/lib/python3.10/dist-packages (from umap-learn>=0.5.0->bertopic) (0.56.4)\n",
            "Requirement already satisfied: pynndescent>=0.5 in /usr/local/lib/python3.10/dist-packages (from umap-learn>=0.5.0->bertopic) (0.5.10)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (3.12.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (2023.4.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (2.27.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (4.5.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (23.1)\n",
            "Requirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.49->umap-learn>=0.5.0->bertopic) (0.39.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from numba>=0.49->umap-learn>=0.5.0->bertopic) (67.7.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas>=1.1.5->bertopic) (1.16.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=0.4.1->bertopic) (1.11.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=0.4.1->bertopic) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=0.4.1->bertopic) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=0.4.1->bertopic) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->sentence-transformers>=0.4.1->bertopic) (3.25.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->sentence-transformers>=0.4.1->bertopic) (16.0.3)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers>=0.4.1->bertopic) (2022.10.31)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers>=0.4.1->bertopic) (0.13.3)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->sentence-transformers>=0.4.1->bertopic) (8.1.3)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision->sentence-transformers>=0.4.1->bertopic) (8.4.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.6.0->sentence-transformers>=0.4.1->bertopic) (2.1.2)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers>=0.4.1->bertopic) (3.4)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.6.0->sentence-transformers>=0.4.1->bertopic) (1.3.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install bertopic"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ont14Lyq4KVw"
      },
      "outputs": [],
      "source": [
        "from bertopic import BERTopic\n",
        "from umap import UMAP\n",
        "from sklearn.model_selection import ParameterSampler\n",
        "\n",
        "# Define the search space for hyperparameters\n",
        "param_grid = {\n",
        "    \"bertopic__nr_topics\": [5, 10, 15, 20],\n",
        "    \"bertopic__min_topic_size\": [5, 10, 15, 20],\n",
        "    \"umap__n_neighbors\": np.arange(5, 50, 5),\n",
        "    \"umap__min_dist\": np.linspace(0.0, 1.0, 11),\n",
        "    \"umap__n_components\": [2, 3, 4, 5],\n",
        "}\n",
        "\n",
        "# Number of random search iterations\n",
        "n_iter = 10\n",
        "\n",
        "def calculate_topic_diversity(topic_words):\n",
        "    unique_words = set()\n",
        "    total_words = 0\n",
        "\n",
        "    for words in topic_words:\n",
        "        unique_words.update(words)\n",
        "        total_words += len(words)\n",
        "\n",
        "    topic_diversity = len(unique_words) / total_words\n",
        "    return topic_diversity\n",
        "\n",
        "# Function to calculate performance metric\n",
        "def evaluate_performance(topics):\n",
        "    # Extract the top n words for each topic\n",
        "    n_words = 10\n",
        "    all_topics = topic_model.get_topics()\n",
        "    topic_words = [words[:n_words] for _, words in all_topics.items()]\n",
        "\n",
        "    # Calculate the topic diversity\n",
        "    topic_diversity = calculate_topic_diversity(topic_words)\n",
        "\n",
        "    return topic_diversity\n",
        "\n",
        "# Random search\n",
        "best_score = -np.inf\n",
        "best_params = None\n",
        "\n",
        "for params in ParameterSampler(param_grid, n_iter):\n",
        "    # Create UMAP and BERTopic instances with sampled hyperparameters\n",
        "    umap_model = UMAP(n_neighbors=params[\"umap__n_neighbors\"],\n",
        "                      min_dist=params[\"umap__min_dist\"],\n",
        "                      n_components=params[\"umap__n_components\"],\n",
        "                      random_state=42)\n",
        "    topic_model = BERTopic(nr_topics=params[\"bertopic__nr_topics\"],\n",
        "                           min_topic_size=params[\"bertopic__min_topic_size\"],\n",
        "                           umap_model=umap_model)\n",
        "\n",
        "    # Fit and transform the data\n",
        "    topics, probabilities = topic_model.fit_transform(data_clean)\n",
        "\n",
        "    # Evaluate performance\n",
        "    score = evaluate_performance(topics)\n",
        "\n",
        "    # Update the best score and best parameters\n",
        "    if score > best_score:\n",
        "        best_score = score\n",
        "        best_params = params\n",
        "\n",
        "# Train the final model with the best hyperparameters\n",
        "umap_model = UMAP(n_neighbors=best_params[\"umap__n_neighbors\"],\n",
        "                  min_dist=best_params[\"umap__min_dist\"],\n",
        "                  n_components=best_params[\"umap__n_components\"],\n",
        "random_state=42)\n",
        "topic_model = BERTopic(nr_topics=best_params[\"bertopic__nr_topics\"],\n",
        "min_topic_size = best_params[\"bertopic__min_topic_size\"], umap_model=umap_model)\n",
        "topics, probabilities = topic_model.fit_transform(data_clean)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eJm5hiZl-cCt",
        "outputId": "c0567daf-14a7-419e-c7b2-dfa0fc79a9b6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1.0\n",
            "{'umap__n_neighbors': 40, 'umap__n_components': 5, 'umap__min_dist': 0.9, 'bertopic__nr_topics': 15, 'bertopic__min_topic_size': 15}\n"
          ]
        }
      ],
      "source": [
        "print(best_score)\n",
        "print(best_params)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fgQXLTM3rDp-",
        "outputId": "6fea1e0f-ad61-49b4-ab49-af1ca057b45a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Topic Diversity:, 0.900000\n"
          ]
        }
      ],
      "source": [
        "from bertopic import BERTopic\n",
        "from umap import UMAP\n",
        "\n",
        "def calculate_topic_diversity(topic_words):\n",
        "    unique_words = set()\n",
        "    total_words = 0\n",
        "\n",
        "    for words in topic_words:\n",
        "        unique_words.update(words)\n",
        "        total_words += len(words)\n",
        "\n",
        "    topic_diversity = len(unique_words) / total_words\n",
        "    return topic_diversity\n",
        "\n",
        "umap_model = UMAP(random_state=42, n_neighbors=40, n_components=5, min_dist=0.9)\n",
        "topic_model = BERTopic(nr_topics=10, umap_model=umap_model)\n",
        "topics, probabilities = topic_model.fit_transform(data_clean)\n",
        "\n",
        "# Extract the top n words for each topic\n",
        "n_words = 10\n",
        "all_topics = topic_model.get_topics()\n",
        "topic_words = [words[:n_words] for _, words in all_topics.items()]\n",
        "\n",
        "# Calculate the topic diversity\n",
        "topic_diversity = calculate_topic_diversity(topic_words)\n",
        "print(f\"Topic Diversity:, {topic_diversity:2f}\")\n",
        "\n",
        "# umap_model = UMAP(random_state=42, n_neighbors = 40, n_components = 5, min_dist = 0.9)\n",
        "# topic_model = BERTopic(nr_topics = 10, umap_model= umap_model)\n",
        "# topics, probabilities = topic_model.fit_transform(data_clean)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 394
        },
        "id": "cJXxSuNsrwwd",
        "outputId": "053c8646-431c-4d80-a483-fe77ceb34733"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-ee9472bd-7139-4857-935b-b329ff8c24e6\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Topic</th>\n",
              "      <th>Count</th>\n",
              "      <th>Name</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1</td>\n",
              "      <td>6006</td>\n",
              "      <td>-1_flight_get_thank_co</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>557</td>\n",
              "      <td>0_bag_baggag_check_lost</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>473</td>\n",
              "      <td>1_cancel_flight_flightl_hold</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2</td>\n",
              "      <td>466</td>\n",
              "      <td>2_call_hold_hour_phone</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3</td>\n",
              "      <td>268</td>\n",
              "      <td>3_delay_flight_hour_late</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>4</td>\n",
              "      <td>221</td>\n",
              "      <td>4_thank_appreci_much_done</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>5</td>\n",
              "      <td>208</td>\n",
              "      <td>5_gate_agent_tarmac_wait</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>6</td>\n",
              "      <td>192</td>\n",
              "      <td>6_servic_custom_terribl_thank</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>7</td>\n",
              "      <td>168</td>\n",
              "      <td>7_dm_follow_sent_thank</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>8</td>\n",
              "      <td>125</td>\n",
              "      <td>8_flight_attend_avgeek_start</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>9</td>\n",
              "      <td>123</td>\n",
              "      <td>9_email_complaint_respons_send</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ee9472bd-7139-4857-935b-b329ff8c24e6')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ee9472bd-7139-4857-935b-b329ff8c24e6 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ee9472bd-7139-4857-935b-b329ff8c24e6');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "    Topic  Count                            Name\n",
              "0  -1      6006   -1_flight_get_thank_co        \n",
              "1   0      557    0_bag_baggag_check_lost       \n",
              "2   1      473    1_cancel_flight_flightl_hold  \n",
              "3   2      466    2_call_hold_hour_phone        \n",
              "4   3      268    3_delay_flight_hour_late      \n",
              "5   4      221    4_thank_appreci_much_done     \n",
              "6   5      208    5_gate_agent_tarmac_wait      \n",
              "7   6      192    6_servic_custom_terribl_thank \n",
              "8   7      168    7_dm_follow_sent_thank        \n",
              "9   8      125    8_flight_attend_avgeek_start  \n",
              "10  9      123    9_email_complaint_respons_send"
            ]
          },
          "execution_count": 617,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Look at distribution of topics by tweet volume\n",
        "topic_model.get_topic_info()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6b3tb11irwwe"
      },
      "source": [
        "topic -1 is a \"misc\" topic"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U2JRvZZFrwwe"
      },
      "source": [
        "#### Need to get a clear understanding of what each topic is about"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 667
        },
        "id": "OUzUpT_Krwwg",
        "outputId": "f1aef563-99cb-438e-ca3f-87740348cd54"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-2.8.3.min.js\"></script>                <div id=\"b1a60b96-530f-4785-a0b9-59897902c0bf\" class=\"plotly-graph-div\" style=\"height:650px; width:650px;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"b1a60b96-530f-4785-a0b9-59897902c0bf\")) {                    Plotly.newPlot(                        \"b1a60b96-530f-4785-a0b9-59897902c0bf\",                        [{\"customdata\":[[0,\"bag | baggag | check | lost | claim\",557],[1,\"cancel | flight | flightl | hold | hour\",473],[2,\"call | hold | hour | phone | wait\",466],[3,\"delay | flight | hour | late | miss\",268],[4,\"thank | appreci | much | done | ok\",221],[5,\"gate | agent | tarmac | wait | sit\",208],[6,\"servic | custom | terribl | thank | line\",192],[7,\"dm | follow | sent | thank | send\",168],[8,\"flight | attend | avgeek | start | er\",125],[9,\"email | complaint | respons | send | receiv\",123]],\"hovertemplate\":\"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#B0BEC5\",\"size\":[557,473,466,268,221,208,192,168,125,123],\"sizemode\":\"area\",\"sizeref\":0.348125,\"symbol\":\"circle\",\"line\":{\"color\":\"DarkSlateGrey\",\"width\":2}},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[13.147577285766602,12.563389778137207,12.02302360534668,-33.39131546020508,12.042997360229492,-33.039791107177734,12.542740821838379,12.359742164611816,-33.74226379394531,12.06607723236084],\"xaxis\":\"x\",\"y\":[-1.8315521478652954,-1.461940884590149,-0.8453070521354675,5.58540678024292,1.2312332391738892,5.233902454376221,-0.6106736660003662,0.8279744982719421,5.936352729797363,0.11590620875358582],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"bar\":[{\"error_x\":{\"color\":\"rgb(36,36,36)\"},\"error_y\":{\"color\":\"rgb(36,36,36)\"},\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"rgb(36,36,36)\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"rgb(36,36,36)\"},\"baxis\":{\"endlinecolor\":\"rgb(36,36,36)\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"rgb(36,36,36)\"},\"type\":\"carpet\"}],\"choropleth\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"type\":\"choropleth\"}],\"contourcarpet\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"type\":\"contourcarpet\"}],\"contour\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"contour\"}],\"heatmapgl\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"heatmapgl\"}],\"heatmap\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"heatmap\"}],\"histogram2dcontour\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"histogram2dcontour\"}],\"histogram2d\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"histogram2d\"}],\"histogram\":[{\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.6}},\"type\":\"histogram\"}],\"mesh3d\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"type\":\"mesh3d\"}],\"parcoords\":[{\"line\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"parcoords\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}],\"scatter3d\":[{\"line\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatter3d\"}],\"scattercarpet\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattercarpet\"}],\"scattergeo\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattergeo\"}],\"scattergl\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattergl\"}],\"scattermapbox\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattermapbox\"}],\"scatterpolargl\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatterpolargl\"}],\"scatterpolar\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatterpolar\"}],\"scatter\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatter\"}],\"scatterternary\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatterternary\"}],\"surface\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"surface\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"rgb(237,237,237)\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"rgb(217,217,217)\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}]},\"layout\":{\"annotationdefaults\":{\"arrowhead\":0,\"arrowwidth\":1},\"autotypenumbers\":\"strict\",\"coloraxis\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"colorscale\":{\"diverging\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"sequential\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"sequentialminus\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]]},\"colorway\":[\"#1F77B4\",\"#FF7F0E\",\"#2CA02C\",\"#D62728\",\"#9467BD\",\"#8C564B\",\"#E377C2\",\"#7F7F7F\",\"#BCBD22\",\"#17BECF\"],\"font\":{\"color\":\"rgb(36,36,36)\"},\"geo\":{\"bgcolor\":\"white\",\"lakecolor\":\"white\",\"landcolor\":\"white\",\"showlakes\":true,\"showland\":true,\"subunitcolor\":\"white\"},\"hoverlabel\":{\"align\":\"left\"},\"hovermode\":\"closest\",\"mapbox\":{\"style\":\"light\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"white\",\"polar\":{\"angularaxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"},\"bgcolor\":\"white\",\"radialaxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"}},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"rgb(232,232,232)\",\"gridwidth\":2,\"linecolor\":\"rgb(36,36,36)\",\"showbackground\":true,\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"},\"yaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"rgb(232,232,232)\",\"gridwidth\":2,\"linecolor\":\"rgb(36,36,36)\",\"showbackground\":true,\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"},\"zaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"rgb(232,232,232)\",\"gridwidth\":2,\"linecolor\":\"rgb(36,36,36)\",\"showbackground\":true,\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"}},\"shapedefaults\":{\"fillcolor\":\"black\",\"line\":{\"width\":0},\"opacity\":0.3},\"ternary\":{\"aaxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"},\"baxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"},\"bgcolor\":\"white\",\"caxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"}},\"title\":{\"x\":0.05},\"xaxis\":{\"automargin\":true,\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"title\":{\"standoff\":15},\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"},\"yaxis\":{\"automargin\":true,\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"title\":{\"standoff\":15},\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"\"},\"visible\":false,\"range\":[-38.80360336303711,15.119713878631591]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"\"},\"visible\":false,\"range\":[-2.1062849700450896,6.826805639266968]},\"legend\":{\"tracegroupgap\":0,\"itemsizing\":\"constant\"},\"margin\":{\"t\":60},\"title\":{\"font\":{\"size\":22,\"color\":\"Black\"},\"text\":\"<b>Intertopic Distance Map\",\"y\":0.95,\"x\":0.5,\"xanchor\":\"center\",\"yanchor\":\"top\"},\"hoverlabel\":{\"font\":{\"size\":16,\"family\":\"Rockwell\"},\"bgcolor\":\"white\"},\"width\":650,\"height\":650,\"sliders\":[{\"active\":0,\"pad\":{\"t\":50},\"steps\":[{\"args\":[{\"marker.color\":[[\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 0\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 1\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 2\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 3\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 4\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 5\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 6\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 7\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\"]]}],\"label\":\"Topic 8\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\"]]}],\"label\":\"Topic 9\",\"method\":\"update\"}]}],\"shapes\":[{\"line\":{\"color\":\"#CFD8DC\",\"width\":2},\"type\":\"line\",\"x0\":-11.84194474220276,\"x1\":-11.84194474220276,\"y0\":-2.1062849700450896,\"y1\":6.826805639266968},{\"line\":{\"color\":\"#9E9E9E\",\"width\":2},\"type\":\"line\",\"x0\":-38.80360336303711,\"x1\":15.119713878631591,\"y0\":2.360260334610939,\"y1\":2.360260334610939}],\"annotations\":[{\"showarrow\":false,\"text\":\"D1\",\"x\":-38.80360336303711,\"y\":2.360260334610939,\"yshift\":10},{\"showarrow\":false,\"text\":\"D2\",\"x\":-11.84194474220276,\"xshift\":10,\"y\":6.826805639266968}]},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('b1a60b96-530f-4785-a0b9-59897902c0bf');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "topic_model.visualize_topics()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KkSyDTwJrwwf",
        "outputId": "e06d91c1-f63d-4014-bfae-cd534a41b9fb"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('bag', 0.2275657901363453),\n",
              " ('baggag', 0.0689740992954628),\n",
              " ('check', 0.04864746368181693),\n",
              " ('lost', 0.047454328583178704),\n",
              " ('claim', 0.02970885466611053),\n",
              " ('still', 0.0289924191915321),\n",
              " ('flight', 0.025701688948328073),\n",
              " ('get', 0.02385863971457976),\n",
              " ('hour', 0.023745778446698847),\n",
              " ('wait', 0.02338716872471915)]"
            ]
          },
          "execution_count": 619,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Understanding what is within each topic\n",
        "topic_model.get_topic(0)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MB_fpGeDvSv1"
      },
      "source": [
        "**TOPIC 0:**\n",
        "\n",
        "This topic appears to be about lost baggage"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TVnnKKLgrwwf",
        "outputId": "2b0ca51f-305c-4e39-9e62-a55ad23c7f0c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('cancel', 0.20761807159125523),\n",
              " ('flight', 0.14731974681639443),\n",
              " ('flightl', 0.13297394726576708),\n",
              " ('hold', 0.045264600719537695),\n",
              " ('hour', 0.03711188888026722),\n",
              " ('rebook', 0.03600187395902681),\n",
              " ('tomorrow', 0.035474732363746174),\n",
              " ('get', 0.034072272589553455),\n",
              " ('help', 0.03368938304261485),\n",
              " ('flightlat', 0.030044320897007717)]"
            ]
          },
          "execution_count": 620,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qO1sBnijsDAy"
      },
      "source": [
        "**TOPIC 1:**\n",
        "\n",
        "This topic appears to be about cancelled flights, rebookings, and refunds"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hdE-ZMWTrwwf",
        "outputId": "1c963e1e-e058-4a39-d838-fb27ecbb0ed4"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('call', 0.11952963672674494),\n",
              " ('hold', 0.10916342857780768),\n",
              " ('hour', 0.07383936350950504),\n",
              " ('phone', 0.06381955141783292),\n",
              " ('wait', 0.05339380147426368),\n",
              " ('minut', 0.045470359188931345),\n",
              " ('get', 0.045041350802122244),\n",
              " ('agent', 0.042675345893144756),\n",
              " ('help', 0.042496234714847175),\n",
              " ('hung', 0.040865011345208944)]"
            ]
          },
          "execution_count": 621,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t0QN4F6F0RS1"
      },
      "source": [
        "**TOPIC 2:**\n",
        "\n",
        "This topic appears to be about phone wait times for customer service calls"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VrGRbEMtrwwf",
        "outputId": "f69bb576-cff9-46ec-96c2-ea0147906292"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('delay', 0.23887322323694457),\n",
              " ('flight', 0.0888511500016484),\n",
              " ('hour', 0.06941161546412397),\n",
              " ('late', 0.05344341855633303),\n",
              " ('miss', 0.05198565317789689),\n",
              " ('connect', 0.04441876042544843),\n",
              " ('ua', 0.04231557786636696),\n",
              " ('time', 0.03997652938553243),\n",
              " ('plane', 0.03231581073002831),\n",
              " ('due', 0.029196007452649095)]"
            ]
          },
          "execution_count": 622,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(3)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1tf4ca-u0Gub"
      },
      "source": [
        "**TOPIC 3:**\n",
        "\n",
        "This topic appears to be about delayed flights and misconnects"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LxEmPtVGrwwf",
        "outputId": "508c592d-2cea-4ce6-f7c1-5011c118f256"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('thank', 0.6075146908106945),\n",
              " ('appreci', 0.21114907931229043),\n",
              " ('much', 0.12587400452143888),\n",
              " ('done', 0.11408714589745099),\n",
              " ('ok', 0.10435302385550355),\n",
              " ('help', 0.08769173420345096),\n",
              " ('awesom', 0.08295987015583574),\n",
              " ('great', 0.07267766013277725),\n",
              " ('guy', 0.06230388525937159),\n",
              " ('updat', 0.05387714873824772)]"
            ]
          },
          "execution_count": 623,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(4)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U_slNu0O0UiZ"
      },
      "source": [
        "**TOPIC 4:**\n",
        "\n",
        "This topic appears to be appreciation/thank you tweets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4GKlgdFLrwwf",
        "outputId": "643a4ae5-0704-46ee-e140-4a372154b6df"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('gate', 0.30286119297161557),\n",
              " ('agent', 0.0738248052391161),\n",
              " ('tarmac', 0.059745124316556986),\n",
              " ('wait', 0.05680935299656468),\n",
              " ('sit', 0.05093319190284553),\n",
              " ('plane', 0.04740666521268563),\n",
              " ('flight', 0.039729144413323085),\n",
              " ('board', 0.03780506484296674),\n",
              " ('min', 0.03751937302097376),\n",
              " ('delay', 0.03723755424339446)]"
            ]
          },
          "execution_count": 624,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(5)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QZn5AgtD3Who"
      },
      "source": [
        "**TOPIC 5:**\n",
        "\n",
        "This topic appears to be about wait times on the tarmac and with boarding at the gate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KtgEZHhx0UiZ",
        "outputId": "188acf36-7c55-45e1-c451-d5a057212c2c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('servic', 0.2732798872746013),\n",
              " ('custom', 0.24084097997540888),\n",
              " ('terribl', 0.0503473068014174),\n",
              " ('thank', 0.04638985871798764),\n",
              " ('line', 0.03546432620564936),\n",
              " ('great', 0.03505055081329756),\n",
              " ('call', 0.0323204860899972),\n",
              " ('rep', 0.03179829903247415),\n",
              " ('help', 0.030529140031537703),\n",
              " ('hold', 0.02878494093265431)]"
            ]
          },
          "execution_count": 625,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(6)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BuBDFHxqvIja"
      },
      "source": [
        "**TOPIC 6:**\n",
        "\n",
        "This topic appears to be about interactions with customer service"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gxqJNgPa0kYU",
        "outputId": "1b8df38a-9455-4bbb-cf82-bb29fe8c34f2"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('dm', 0.6712099076504824),\n",
              " ('follow', 0.3933447213906641),\n",
              " ('sent', 0.2981806758929228),\n",
              " ('thank', 0.11899735912946047),\n",
              " ('send', 0.1076240352636655),\n",
              " ('pleas', 0.0730054882287865),\n",
              " ('back', 0.06428433611679904),\n",
              " ('need', 0.05077781842126417),\n",
              " ('order', 0.04837136562129898),\n",
              " ('info', 0.04259835203968909)]"
            ]
          },
          "execution_count": 626,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(7)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dceKLHKdvU-h"
      },
      "source": [
        "**TOPIC 7:**\n",
        "\n",
        "This topic appears to be about twitter specific actions these customers are taking (dming, following, sending info)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NTxQXle402XI",
        "outputId": "56b01efc-c46d-40d4-e0e9-428ffe680f96"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('flight', 0.11855775576737536),\n",
              " ('attend', 0.05260275941221941),\n",
              " ('avgeek', 0.05149601084949737),\n",
              " ('start', 0.042598130747012554),\n",
              " ('er', 0.038963622095124356),\n",
              " ('airport', 0.03851356384697827),\n",
              " ('thank', 0.0367777411258134),\n",
              " ('fli', 0.03261411427364624),\n",
              " ('aug', 0.032469685079270304),\n",
              " ('daili', 0.032469685079270304)]"
            ]
          },
          "execution_count": 627,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(8)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qtjt63Lr0kYU"
      },
      "source": [
        "**TOPIC 8:**\n",
        "\n",
        "This topic appears to be about the People (flight attendants, crew, gate agents)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9h1Awv6l47Yp",
        "outputId": "019d6931-0829-4fda-a9ba-6728758235d0"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('email', 0.2467589031302336),\n",
              " ('complaint', 0.12767416365040402),\n",
              " ('respons', 0.05964201173238784),\n",
              " ('send', 0.05434843244314402),\n",
              " ('receiv', 0.04993739396412991),\n",
              " ('custom', 0.048514261793879625),\n",
              " ('relat', 0.04637241649121777),\n",
              " ('mail', 0.045469346888890524),\n",
              " ('address', 0.041292693876651576),\n",
              " ('sent', 0.03775854896556881)]"
            ]
          },
          "execution_count": 628,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(9)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YCo0Sesw47Yp"
      },
      "source": [
        "**TOPIC 9:**\n",
        "\n",
        "This topic appears to be about filing a complaint"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HZuoZenwrwwg",
        "outputId": "d102b0d0-7b12-4ab4-e326-5ac6eb54d9c2"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{4: ['thank ',\n",
              "  'thank ',\n",
              "  'thank ',\n",
              "  'wow help ',\n",
              "  'help ',\n",
              "  'nice work updat ',\n",
              "  'done ',\n",
              "  'done hope offer someth ',\n",
              "  'done thank ',\n",
              "  'okay thank ',\n",
              "  'great thank ',\n",
              "  'good thank ',\n",
              "  'gracia appreci ',\n",
              "  'thank respons much appreci ',\n",
              "  'thank much appreci help '],\n",
              " 5: ['gate agent dfw gate pathet tonight terribl commun stori chang apathet ',\n",
              "  'ground crew inept left truck sit tarmac gate agent got gate ',\n",
              "  'watch connect flight den pull away gate pull gate slapinthefac '],\n",
              " 9: ['come one answer call email ',\n",
              "  'go file formal complaint never happen pleas direct us right depart ',\n",
              "  'social media team said sorri thank submit complaint got resolut amaz rep call ',\n",
              "  'email phone call noth screw flight famili friend flight cancel flight reserv ',\n",
              "  'cancel flight flight send email say think next flight custom outrag http co kkwiwi ',\n",
              "  'email call past month flight chang help wish would sent cancel flightlat yesterday ',\n",
              "  'fill whole form receiv poor poor respons direct email address repli ',\n",
              "  'thank respons twitter truli impress ',\n",
              "  'fantast thank tri tweet photo view '],\n",
              " 0: ['usairway help find baggag repli asap direct detail ',\n",
              "  'check boardingpass checkin bagag guess set ',\n",
              "  'thank noth next time bring whole extra suitcas extra pound make sens right ',\n",
              "  'describ fece sprinkl bathroom inconveni help might find baggag lost amp return ',\n",
              "  'noth ruin wet frustrat travel hour chang clean cloth ',\n",
              "  'hdsportsguy ye hope updat soon need cloth meet tomorrow '],\n",
              " 3: ['evri st flight take delay amp evri nd time miss evri time breakup ',\n",
              "  'feel well inform ye flight inde delay thank http co fuchzrzjg ',\n",
              "  'flight chi buf last night almost hour delay cuz someon got plane without pass sound safe ',\n",
              "  'flt ord correct per dm flight ord cvg charg incorrectli ',\n",
              "  'flight delay keep chang delay delay checkin time late flightr http co ntuix dbyr ',\n",
              "  'know bad pray connect flight delay hour delay first flight justgetmehom ',\n",
              "  'delay two hour ord gt cdg one time need catch train ',\n",
              "  'chanc make connect el paso aa dfw sat aa delay minut ',\n",
              "  'flight delay due mainten lit caus miss connect iah msi help ',\n",
              "  'big compani happi someth custom cost delay oh mention ',\n",
              "  'make delay ',\n",
              "  'suck hour delay ',\n",
              "  'spend hotel weather delay weather someth complet control ',\n",
              "  'blame weather lol wind cloud sky delay way ru kid charg lb delay ',\n",
              "  'btw weather delay beauti weather sunni california nolov noexcus cali http co kumtbger '],\n",
              " 8: ['flight na bwi ton march ',\n",
              "  'favorit airlin wait book flight new orlean trip april ',\n",
              "  'unveil new flight includ ohio http co grkbj bxlk '],\n",
              " 2: ['solut ',\n",
              "  'thank info number call speak person go take hour type ',\n",
              "  'system alway hang ',\n",
              "  'alreadi follow one number agent rebook went ugh ',\n",
              "  'help need speak live agent lose onlin reserv held ',\n",
              "  'websit allow post requir document keep get technic difficulti '],\n",
              " 1: ['thank flight oper cancel flightl ',\n",
              "  'stop cancel flight flight us histori test take chemistri thing learn understand ',\n",
              "  'concern cancel flight flight bna tomorrow ',\n",
              "  'flight nashvil cancel flightl anoth flight tri call amp cannot get human ',\n",
              "  'flight cancel flight delay expect full compens park ticket hotel amp flight ',\n",
              "  'reschedul return flight japan uh tri call unitedairlin customerservic min wait fun ',\n",
              "  'flight cancel flightl one week custom servic ',\n",
              "  'worst flight custom servic second time cancel flight gotta pay hotel find ride ',\n",
              "  'want know regard connect flight sinc st cancel flightl could assist '],\n",
              " 6: ['thank supervisor miriam provid great custom servic final ',\n",
              "  'thank secur room night thx help custom servic rep dorn thank ',\n",
              "  'yet next termin custom servic area ',\n",
              "  'wonder reason custom servic busi hold past day mani hour answer help plz ',\n",
              "  'custom servic suck hang wait hour talk minut ',\n",
              "  'speak custom servic hour still hold guy work nogood ',\n",
              "  'money suck airlin terribl terribl custom servic ',\n",
              "  'hold airlin min count servic terribl ',\n",
              "  'give servic terribl forc use certain rout never would ',\n",
              "  'agent servic weekend trip amaz thank ',\n",
              "  'claudia iah termin travel assist set great servic thank ',\n",
              "  'thank quick custom servic today know winter weather fault ',\n",
              "  'expect higher level custom servic amp flight crew give us accur inform ',\n",
              "  'mean beyond opaqu mainten issu ye would basic custom servic ewr staff manag ',\n",
              "  'line sfo custom servic rude disorgan congest dissapoint '],\n",
              " 7: ['dm ',\n",
              "  'dm anyth ',\n",
              "  'let dm fsz yo ',\n",
              "  'follow back dm guy ',\n",
              "  'cannot dm follow ',\n",
              "  'dm follow ',\n",
              "  'sent ',\n",
              "  'sent ',\n",
              "  'sent ',\n",
              "  'follow ',\n",
              "  'got follow page ',\n",
              "  'would need follow first ',\n",
              "  'sent dm ',\n",
              "  'thx follow sent dm ',\n",
              "  'sent dm ']}"
            ]
          },
          "execution_count": 629,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#the most representative tweets within each topic:\n",
        "topic_model.get_representative_docs()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 581
        },
        "id": "ZRg5etyy-VFN",
        "outputId": "312e53bf-8624-4b71-b427-d200658fff5e"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-75746b79-40c0-4a5a-acbd-242c99639cbe\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>airline</th>\n",
              "      <th>name</th>\n",
              "      <th>text</th>\n",
              "      <th>tweet_created</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>topics</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>United</td>\n",
              "      <td>rdowning76</td>\n",
              "      <td>united thanks</td>\n",
              "      <td>2015-02-24 11:42:48</td>\n",
              "      <td>POSITIVE</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>10</td>\n",
              "      <td>United</td>\n",
              "      <td>slandail</td>\n",
              "      <td>united gate agent hooked me up with alternate flights. if you have a way to prevent the constant issues that would rock.</td>\n",
              "      <td>2015-02-24 11:19:32</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>27</td>\n",
              "      <td>United</td>\n",
              "      <td>nate2482</td>\n",
              "      <td>united the internet is a great thing. i am emailing executives in your company maybe they will respond to me in a timely manner.</td>\n",
              "      <td>2015-02-24 10:47:29</td>\n",
              "      <td>POSITIVE</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>34</td>\n",
              "      <td>United</td>\n",
              "      <td>nate2482</td>\n",
              "      <td>united i sure did. i had to drive a total of hours to get my own bag. i d like to explain that debacle but no one wants to talk to me.</td>\n",
              "      <td>2015-02-24 10:35:42</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>36</td>\n",
              "      <td>United</td>\n",
              "      <td>BocheBillions</td>\n",
              "      <td>united see we were told repeatedly that the pilot was late flight and kept getting late flightr. after we boarded there was a defibrillator issue.</td>\n",
              "      <td>2015-02-24 10:30:53</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>37</td>\n",
              "      <td>United</td>\n",
              "      <td>CoachJoette</td>\n",
              "      <td>united why no preferred security line anymore. my tsa pre check didn t pull on my silverairways partner flight platinum spoiled</td>\n",
              "      <td>2015-02-24 10:30:00</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>38</td>\n",
              "      <td>United</td>\n",
              "      <td>lotusfitness</td>\n",
              "      <td>united newarkliberty airport need to indicate different gate s for terminals a amp c. flyingainteasy</td>\n",
              "      <td>2015-02-24 10:25:12</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>42</td>\n",
              "      <td>United</td>\n",
              "      <td>OohLaLa09</td>\n",
              "      <td>united yes we ve been with the agents for the last minutes. one of the agents have been very rude but thankfully ladan has been nice.</td>\n",
              "      <td>2015-02-24 10:16:16</td>\n",
              "      <td>POSITIVE</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>43</td>\n",
              "      <td>United</td>\n",
              "      <td>cehertz</td>\n",
              "      <td>united that s right with an overnight miami.</td>\n",
              "      <td>2015-02-24 10:14:19</td>\n",
              "      <td>POSITIVE</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53</th>\n",
              "      <td>54</td>\n",
              "      <td>United</td>\n",
              "      <td>kevinfla3</td>\n",
              "      <td>united well sitting on the ground on time but waiting for a gate....again tiredofthis</td>\n",
              "      <td>2015-02-24 09:54:12</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-75746b79-40c0-4a5a-acbd-242c99639cbe')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-75746b79-40c0-4a5a-acbd-242c99639cbe button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-75746b79-40c0-4a5a-acbd-242c99639cbe');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "    tweet_id airline           name  \\\n",
              "0   1         United  rdowning76      \n",
              "9   10        United  slandail        \n",
              "26  27        United  nate2482        \n",
              "33  34        United  nate2482        \n",
              "35  36        United  BocheBillions   \n",
              "36  37        United  CoachJoette     \n",
              "37  38        United  lotusfitness    \n",
              "41  42        United  OohLaLa09       \n",
              "42  43        United  cehertz         \n",
              "53  54        United  kevinfla3       \n",
              "\n",
              "                                                                                                                                                  text  \\\n",
              "0   united thanks                                                                                                                                        \n",
              "9   united gate agent hooked me up with alternate flights. if you have a way to prevent the constant issues that would rock.                             \n",
              "26  united the internet is a great thing. i am emailing executives in your company maybe they will respond to me in a timely manner.                     \n",
              "33  united i sure did. i had to drive a total of hours to get my own bag. i d like to explain that debacle but no one wants to talk to me.               \n",
              "35  united see we were told repeatedly that the pilot was late flight and kept getting late flightr. after we boarded there was a defibrillator issue.   \n",
              "36  united why no preferred security line anymore. my tsa pre check didn t pull on my silverairways partner flight platinum spoiled                      \n",
              "37  united newarkliberty airport need to indicate different gate s for terminals a amp c. flyingainteasy                                                 \n",
              "41  united yes we ve been with the agents for the last minutes. one of the agents have been very rude but thankfully ladan has been nice.                \n",
              "42  united that s right with an overnight miami.                                                                                                         \n",
              "53  united well sitting on the ground on time but waiting for a gate....again tiredofthis                                                                \n",
              "\n",
              "         tweet_created sentiment  topics  \n",
              "0  2015-02-24 11:42:48  POSITIVE  4       \n",
              "9  2015-02-24 11:19:32  NEGATIVE  5       \n",
              "26 2015-02-24 10:47:29  POSITIVE  9       \n",
              "33 2015-02-24 10:35:42  NEGATIVE  0       \n",
              "35 2015-02-24 10:30:53  NEGATIVE  3       \n",
              "36 2015-02-24 10:30:00  NEGATIVE  8       \n",
              "37 2015-02-24 10:25:12  NEGATIVE  5       \n",
              "41 2015-02-24 10:16:16  POSITIVE  2       \n",
              "42 2015-02-24 10:14:19  POSITIVE  8       \n",
              "53 2015-02-24 09:54:12  NEGATIVE  5       "
            ]
          },
          "execution_count": 630,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "tweets['topics'] = topics\n",
        "non_misc = tweets[tweets['topics'] != -1]\n",
        "non_misc.head(10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5zpoBuOyBshw"
      },
      "source": [
        "Developing an understanding of what are the hot topics in the airline industry. It will be crucial to understand where the client sits with customer statisfaction and sentiment against this baseline."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n3NeRLktyAPO"
      },
      "source": [
        "**ALL AIRLINES:**\n",
        "\n",
        "**TOPIC 0:**\n",
        "This topic appears to be about lost baggage\n",
        " \n",
        "**TOPIC 1:**\n",
        "This topic appears to be about cancelled flights, rebookings, and refunds\n",
        " \n",
        "**TOPIC 2:**\n",
        "This topic appears to be about phone wait times for customer service calls\n",
        " \n",
        "**TOPIC 3:**\n",
        "This topic appears to be about delayed flights and misconnects\n",
        "\n",
        "**TOPIC 4:**\n",
        "This topic appears to be appreciation/thank you tweets\n",
        "\n",
        "**TOPIC 5:**\n",
        "This topic appears to be about wait times on the tarmac and with boarding at the gate\n",
        "\n",
        "**TOPIC 6:**\n",
        "This topic appears to be about interactions with customer service\n",
        "\n",
        "**TOPIC 7:**\n",
        "This topic appears to be about twitter specific actions these customers are taking (dming, following, sending info)\n",
        "\n",
        "**TOPIC 8:**\n",
        "This topic appears to be about the People (flight attendants, crew, gate agents)\n",
        "\n",
        "**TOPIC 9:**\n",
        "This topic appears to be about filing a complaint\n",
        " \n",
        " \n",
        " \n",
        " \n",
        " \n",
        " \n",
        " \n",
        " \n",
        " \n",
        " \n",
        " \n",
        " \n",
        " \n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JnxAxhUSq-xp"
      },
      "source": [
        "##Topic Modeling & Sentiment Analysis - Airline Specific"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DbKoas5Qo2yM"
      },
      "source": [
        "Here I run topic modeling on each airline to understand what are the big topics (top 6) for each airline to understand what the client's customers are talking about for each of the airlines. Then I plan to run an in-depth sentiment analysis on each of the topics to see what the client (United) could do better, and where they are edging out their competition and can futher expand on. This will map out the differentiators to competition, both good and not so good"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "az9Jk7ORpXfH"
      },
      "outputs": [],
      "source": [
        "UAL_tweets = tweets[tweets['airline'] == 'United']\n",
        "AA_tweets = tweets[tweets['airline'] == 'American']\n",
        "SW_tweets = tweets[tweets['airline'] == 'Southwest']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rPXIJTdi8GHG",
        "outputId": "cebcbf6c-8153-45cd-a8b4-cb569336d603"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "execution_count": 53,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "nltk.download('stopwords')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sj48CeIEq5zQ"
      },
      "outputs": [],
      "source": [
        "#Removing stopwords:\n",
        "\n",
        "from nltk.corpus import stopwords\n",
        "\n",
        "\n",
        "#UAL\n",
        "UAL_data_list = UAL_tweets['text'].tolist()        \n",
        "\n",
        "add_stopwords = [\"united\"]\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english')))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "UAL_data_nostops = remove_stopwords(UAL_data_list)\n",
        "\n",
        "#AA\n",
        "AA_data_list = AA_tweets['text'].tolist()        \n",
        "\n",
        "add_stopwords = [\"americanair\"]\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english')))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "AA_data_nostops = remove_stopwords(AA_data_list)\n",
        "\n",
        "#SW\n",
        "SW_data_list = SW_tweets['text'].tolist()        \n",
        "\n",
        "add_stopwords = [\"southwestair\"]\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english')))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "SW_data_nostops = remove_stopwords(SW_data_list)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nSRx4PhLreiK"
      },
      "outputs": [],
      "source": [
        "#Stemming\n",
        "from nltk.stem import PorterStemmer \n",
        "ps = PorterStemmer()\n",
        " \n",
        "#UAL\n",
        "# choose some words to be stemmed\n",
        "UAL_data_clean = []\n",
        "\n",
        "for i in UAL_data_nostops:\n",
        "    string = ''\n",
        "    \n",
        "    for w in i:\n",
        "        \n",
        "        a = ps.stem(w)+\" \"\n",
        "        \n",
        "        string += a\n",
        "    UAL_data_clean.append(string)\n",
        "\n",
        "\n",
        "#AA\n",
        "# choose some words to be stemmed\n",
        "AA_data_clean = []\n",
        "\n",
        "for i in AA_data_nostops:\n",
        "    string = ''\n",
        "    \n",
        "    for w in i:\n",
        "        \n",
        "        a = ps.stem(w)+\" \"\n",
        "        \n",
        "        string += a\n",
        "    AA_data_clean.append(string)\n",
        "\n",
        "#SW\n",
        "# choose some words to be stemmed\n",
        "SW_data_clean = []\n",
        "\n",
        "for i in SW_data_nostops:\n",
        "    string = ''\n",
        "    \n",
        "    for w in i:\n",
        "        \n",
        "        a = ps.stem(w)+\" \"\n",
        "        \n",
        "        string += a\n",
        "    SW_data_clean.append(string)\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QS3YQU-_OkDe"
      },
      "source": [
        "### Topic detection for United"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ibgJe3trOkDe"
      },
      "outputs": [],
      "source": [
        "from bertopic import BERTopic\n",
        "\n",
        "topic_model = BERTopic(nr_topics = 6)\n",
        "\n",
        "UAL_topics, UAL_probabilities = topic_model.fit_transform(UAL_data_clean)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "id": "_OwguMWwOkDe",
        "outputId": "78454939-e1db-4f29-f7ca-479abfebe273"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-484de045-6aa2-466b-8598-4f68ab6b3b1b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Topic</th>\n",
              "      <th>Count</th>\n",
              "      <th>Name</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1</td>\n",
              "      <td>2727</td>\n",
              "      <td>-1_flight_get_co_http</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>331</td>\n",
              "      <td>0_bag_baggag_check_lost</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>227</td>\n",
              "      <td>1_thank_flight_plane_appreci</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2</td>\n",
              "      <td>153</td>\n",
              "      <td>2_cancel_flight_flightl_get</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3</td>\n",
              "      <td>135</td>\n",
              "      <td>3_custom_servic_care_thank</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>4</td>\n",
              "      <td>129</td>\n",
              "      <td>4_delay_flight_hour_time</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>5</td>\n",
              "      <td>103</td>\n",
              "      <td>5_gate_agent_wait_sit</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-484de045-6aa2-466b-8598-4f68ab6b3b1b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-484de045-6aa2-466b-8598-4f68ab6b3b1b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-484de045-6aa2-466b-8598-4f68ab6b3b1b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "   Topic  Count                          Name\n",
              "0 -1      2727   -1_flight_get_co_http       \n",
              "1  0      331    0_bag_baggag_check_lost     \n",
              "2  1      227    1_thank_flight_plane_appreci\n",
              "3  2      153    2_cancel_flight_flightl_get \n",
              "4  3      135    3_custom_servic_care_thank  \n",
              "5  4      129    4_delay_flight_hour_time    \n",
              "6  5      103    5_gate_agent_wait_sit       "
            ]
          },
          "execution_count": 635,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic_info()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "huM6NrSztg54"
      },
      "source": [
        "####Need to get a clear understanding of what each topic is about"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 667
        },
        "id": "SFKve9Qstw50",
        "outputId": "b20c0f24-5956-4ffd-9a8b-d563bff1caa2"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-2.8.3.min.js\"></script>                <div id=\"06b085d4-c790-4f8e-9398-51098005cbc2\" class=\"plotly-graph-div\" style=\"height:650px; width:650px;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"06b085d4-c790-4f8e-9398-51098005cbc2\")) {                    Plotly.newPlot(                        \"06b085d4-c790-4f8e-9398-51098005cbc2\",                        [{\"customdata\":[[0,\"bag | baggag | check | lost | flight\",331],[1,\"thank | flight | plane | appreci | great\",227],[2,\"cancel | flight | flightl | get | delay\",153],[3,\"custom | servic | care | thank | contact\",135],[4,\"delay | flight | hour | time | miss\",129],[5,\"gate | agent | wait | sit | min\",103]],\"hovertemplate\":\"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#B0BEC5\",\"size\":[331,227,153,135,129,103],\"sizemode\":\"area\",\"sizeref\":0.206875,\"symbol\":\"circle\",\"line\":{\"color\":\"DarkSlateGrey\",\"width\":2}},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[7.9343156814575195,8.239672660827637,8.697415351867676,7.658430099487305,5.087434768676758,4.727530002593994],\"xaxis\":\"x\",\"y\":[14.789937019348145,15.38817310333252,14.52623176574707,14.213632583618164,0.13492737710475922,0.4945102632045746],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"bar\":[{\"error_x\":{\"color\":\"rgb(36,36,36)\"},\"error_y\":{\"color\":\"rgb(36,36,36)\"},\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"rgb(36,36,36)\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"rgb(36,36,36)\"},\"baxis\":{\"endlinecolor\":\"rgb(36,36,36)\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"rgb(36,36,36)\"},\"type\":\"carpet\"}],\"choropleth\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"type\":\"choropleth\"}],\"contourcarpet\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"type\":\"contourcarpet\"}],\"contour\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"contour\"}],\"heatmapgl\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"heatmapgl\"}],\"heatmap\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"heatmap\"}],\"histogram2dcontour\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"histogram2dcontour\"}],\"histogram2d\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"histogram2d\"}],\"histogram\":[{\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.6}},\"type\":\"histogram\"}],\"mesh3d\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"type\":\"mesh3d\"}],\"parcoords\":[{\"line\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"parcoords\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}],\"scatter3d\":[{\"line\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatter3d\"}],\"scattercarpet\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattercarpet\"}],\"scattergeo\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattergeo\"}],\"scattergl\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattergl\"}],\"scattermapbox\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattermapbox\"}],\"scatterpolargl\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatterpolargl\"}],\"scatterpolar\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatterpolar\"}],\"scatter\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatter\"}],\"scatterternary\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatterternary\"}],\"surface\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"surface\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"rgb(237,237,237)\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"rgb(217,217,217)\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}]},\"layout\":{\"annotationdefaults\":{\"arrowhead\":0,\"arrowwidth\":1},\"autotypenumbers\":\"strict\",\"coloraxis\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"colorscale\":{\"diverging\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"sequential\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"sequentialminus\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]]},\"colorway\":[\"#1F77B4\",\"#FF7F0E\",\"#2CA02C\",\"#D62728\",\"#9467BD\",\"#8C564B\",\"#E377C2\",\"#7F7F7F\",\"#BCBD22\",\"#17BECF\"],\"font\":{\"color\":\"rgb(36,36,36)\"},\"geo\":{\"bgcolor\":\"white\",\"lakecolor\":\"white\",\"landcolor\":\"white\",\"showlakes\":true,\"showland\":true,\"subunitcolor\":\"white\"},\"hoverlabel\":{\"align\":\"left\"},\"hovermode\":\"closest\",\"mapbox\":{\"style\":\"light\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"white\",\"polar\":{\"angularaxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"},\"bgcolor\":\"white\",\"radialaxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"}},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"rgb(232,232,232)\",\"gridwidth\":2,\"linecolor\":\"rgb(36,36,36)\",\"showbackground\":true,\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"},\"yaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"rgb(232,232,232)\",\"gridwidth\":2,\"linecolor\":\"rgb(36,36,36)\",\"showbackground\":true,\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"},\"zaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"rgb(232,232,232)\",\"gridwidth\":2,\"linecolor\":\"rgb(36,36,36)\",\"showbackground\":true,\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"}},\"shapedefaults\":{\"fillcolor\":\"black\",\"line\":{\"width\":0},\"opacity\":0.3},\"ternary\":{\"aaxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"},\"baxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"},\"bgcolor\":\"white\",\"caxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"}},\"title\":{\"x\":0.05},\"xaxis\":{\"automargin\":true,\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"title\":{\"standoff\":15},\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"},\"yaxis\":{\"automargin\":true,\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"title\":{\"standoff\":15},\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"\"},\"visible\":false,\"range\":[4.018400502204895,10.002027654647828]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"\"},\"visible\":false,\"range\":[0.11468827053904533,17.6963990688324]},\"legend\":{\"tracegroupgap\":0,\"itemsizing\":\"constant\"},\"margin\":{\"t\":60},\"title\":{\"font\":{\"size\":22,\"color\":\"Black\"},\"text\":\"<b>Intertopic Distance Map\",\"y\":0.95,\"x\":0.5,\"xanchor\":\"center\",\"yanchor\":\"top\"},\"hoverlabel\":{\"font\":{\"size\":16,\"family\":\"Rockwell\"},\"bgcolor\":\"white\"},\"width\":650,\"height\":650,\"sliders\":[{\"active\":0,\"pad\":{\"t\":50},\"steps\":[{\"args\":[{\"marker.color\":[[\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 0\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 1\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 2\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 3\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\"]]}],\"label\":\"Topic 4\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\"]]}],\"label\":\"Topic 5\",\"method\":\"update\"}]}],\"shapes\":[{\"line\":{\"color\":\"#CFD8DC\",\"width\":2},\"type\":\"line\",\"x0\":7.010214078426362,\"x1\":7.010214078426362,\"y0\":0.11468827053904533,\"y1\":17.6963990688324},{\"line\":{\"color\":\"#9E9E9E\",\"width\":2},\"type\":\"line\",\"x0\":4.018400502204895,\"x1\":10.002027654647828,\"y0\":8.905543669685722,\"y1\":8.905543669685722}],\"annotations\":[{\"showarrow\":false,\"text\":\"D1\",\"x\":4.018400502204895,\"y\":8.905543669685722,\"yshift\":10},{\"showarrow\":false,\"text\":\"D2\",\"x\":7.010214078426362,\"xshift\":10,\"y\":17.6963990688324}]},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('06b085d4-c790-4f8e-9398-51098005cbc2');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "topic_model.visualize_topics()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GA38hO4lujyQ",
        "outputId": "424c721d-0ae4-4271-c288-62e2a151580d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('bag', 0.2387429316557421),\n",
              " ('baggag', 0.06714015917348809),\n",
              " ('check', 0.06119789583610768),\n",
              " ('lost', 0.045477137759974966),\n",
              " ('flight', 0.04100276014514238),\n",
              " ('get', 0.03162262256245142),\n",
              " ('still', 0.03117687656611434),\n",
              " ('claim', 0.02758109494789827),\n",
              " ('wait', 0.026199188618698698),\n",
              " ('hour', 0.024832624118016944)]"
            ]
          },
          "execution_count": 637,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Understanding what is within each topic\n",
        "topic_model.get_topic(0)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DuoHRmxtv2Nr"
      },
      "source": [
        "**Topic 0**\n",
        "\n",
        "Topic seems to be about lost baggage"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DvFm8WJ0vzQX",
        "outputId": "eefe0135-3d65-4279-fdd4-002ffa0b16ae"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('thank', 0.21233158039581296),\n",
              " ('flight', 0.12785516373186664),\n",
              " ('plane', 0.1076923752049972),\n",
              " ('appreci', 0.0679108727928005),\n",
              " ('great', 0.04337680169304061),\n",
              " ('help', 0.041157621194199065),\n",
              " ('ua', 0.03961257969661268),\n",
              " ('attend', 0.03150979438042774),\n",
              " ('good', 0.031417788855105405),\n",
              " ('okay', 0.031271431631108355)]"
            ]
          },
          "execution_count": 638,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DDw3c8jjv3hz"
      },
      "source": [
        "**Topic 1**\n",
        "\n",
        "Topic seems appreciation tweets/thank yous"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dHK4MP3MwDcB",
        "outputId": "6bae63d1-74fa-45ac-bc57-a13b8fae8a7c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('cancel', 0.271955789738553),\n",
              " ('flight', 0.1863626132062938),\n",
              " ('flightl', 0.16508265019576096),\n",
              " ('get', 0.03458407714752466),\n",
              " ('delay', 0.030154098235779532),\n",
              " ('book', 0.026972275535419463),\n",
              " ('two', 0.025542732925730498),\n",
              " ('last', 0.02484930488667062),\n",
              " ('due', 0.02445345277295488),\n",
              " ('flightlat', 0.023883349873256153)]"
            ]
          },
          "execution_count": 639,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(2)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iMccedW_wDcC"
      },
      "source": [
        "**Topic 2**\n",
        "\n",
        "Topic seems to be about cancellations and rebookings"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "akvpUBvVwuUA",
        "outputId": "fa8994d5-6fa6-4733-c69d-2112424de499"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('custom', 0.2836239129962077),\n",
              " ('servic', 0.2669799916036024),\n",
              " ('care', 0.08917118207295835),\n",
              " ('thank', 0.05539239994211327),\n",
              " ('contact', 0.041868903973473086),\n",
              " ('phone', 0.03936530249162366),\n",
              " ('respons', 0.03602826826124666),\n",
              " ('help', 0.03577303232647339),\n",
              " ('rep', 0.034582951976819695),\n",
              " ('worst', 0.030755992336645544)]"
            ]
          },
          "execution_count": 640,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(3)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iFj7nOoXwxaJ"
      },
      "source": [
        "**Topic 3**\n",
        "\n",
        "Topic seems to be about contacting customer service"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qgm1pv9fw3Hz",
        "outputId": "bda8d132-3fdb-47c3-8a14-351348671183"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('delay', 0.27081464568441616),\n",
              " ('flight', 0.13001774543193265),\n",
              " ('hour', 0.09032289375341879),\n",
              " ('time', 0.05946236564723379),\n",
              " ('miss', 0.058459778397971424),\n",
              " ('connect', 0.05320480879506004),\n",
              " ('late', 0.051465126508332563),\n",
              " ('ua', 0.049497572744759286),\n",
              " ('wait', 0.045552365488026964),\n",
              " ('crew', 0.035566344231511844)]"
            ]
          },
          "execution_count": 641,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(4)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7-cxwC9lxFIa"
      },
      "source": [
        "**Topic 4**\n",
        "\n",
        "Topic seems to be about delays and misconnects"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hnLqcQFSyJ4l",
        "outputId": "1160cb78-a07d-47d6-edd5-ee576408d639"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('gate', 0.3465557240475961),\n",
              " ('agent', 0.07369719122516827),\n",
              " ('wait', 0.06970459439343271),\n",
              " ('sit', 0.06385565726664691),\n",
              " ('min', 0.05356782055173048),\n",
              " ('time', 0.05203763795895664),\n",
              " ('flight', 0.04732158604006749),\n",
              " ('tarmac', 0.04623554571432268),\n",
              " ('board', 0.0456220707584375),\n",
              " ('ua', 0.04149321455583726)]"
            ]
          },
          "execution_count": 642,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(5)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S0O7XLVVyL85"
      },
      "source": [
        "**Topic 5**\n",
        "\n",
        "Topic seems to be about the gate agents, delays at the gate, and while taxiing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fYBg6QjXvb6Y",
        "outputId": "ead8c234-0276-474a-b95d-e56a6485405e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{1: ['thank ',\n",
              "  'thank ',\n",
              "  'thank ',\n",
              "  'flight thx ',\n",
              "  'flight ',\n",
              "  'flight ',\n",
              "  'thank got straightend last night ',\n",
              "  'thank jt ',\n",
              "  'thx updat ',\n",
              "  'sent ',\n",
              "  'sent thank ',\n",
              "  'thank offer sort ',\n",
              "  'perfect thank ',\n",
              "  'great thank ',\n",
              "  'cool thank ',\n",
              "  'thank appreci respons ',\n",
              "  'thank listen compliant right thing appreci work ',\n",
              "  'appreci effort get home '],\n",
              " 3: ['submit custom care form jan th still respons littl longer busi day ',\n",
              "  'custom care email phone rep speak mani issu place charact ',\n",
              "  'thank repli email custom care depart experi ',\n",
              "  'custom servic solut might miss show ',\n",
              "  'stellar custom servic earn busi attent detail ',\n",
              "  'thank abl app still custom servic seen yyz '],\n",
              " 5: ['phenomen frm gate sea gate iad ',\n",
              "  'made gate let us http co xatoxbnsfa ',\n",
              "  'leav tarmac gate agent cannot agre head count min count peopl max epicfailunit '],\n",
              " 0: ['pm bag still even courtesi toiletri kit flight experi expect disappoint ',\n",
              "  'caught earlier flight ord gate check bag lost hare origin flight land minut frustrat ',\n",
              "  'receiv bag last night still miss one '],\n",
              " 2: ['check see flight atl ord got cancel flight ',\n",
              "  'den phx flight tomorrow cancel flight ask overnight nite lax sna told without pay wrong ',\n",
              "  'cancel flightl flight without notif ',\n",
              "  'well degre could know experi cancel flight mile card thank god delta ',\n",
              "  'flight bgm cancel flightl last night get hotel voucher pay cab back wors cab experi ever ',\n",
              "  'receiv hotel food also email mile last trip hr delay mile less cancel flightlat '],\n",
              " 4: ['never flight delay hour due unbalanc load delay flylaxairport great job idiot ',\n",
              "  'flight ua sfo burbank delay twice still plane sight get togeth ',\n",
              "  'flight delay three hour pilot never show ',\n",
              "  'charg delay still wait ',\n",
              "  'delay cs help notif ',\n",
              "  'suck hour delay ']}"
            ]
          },
          "execution_count": 643,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#the most representative tweets within each topic:\n",
        "topic_model.get_representative_docs()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "id": "t5HHxvFq0cco",
        "outputId": "02eaaa62-41c9-4ec0-fac3-5ab4e91b6c56"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-b44f59ad-737e-4ab2-8188-9fe67008580e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Topic</th>\n",
              "      <th>Count</th>\n",
              "      <th>Name</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1</td>\n",
              "      <td>2727</td>\n",
              "      <td>-1_flight_get_co_http</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>331</td>\n",
              "      <td>0_bag_baggag_check_lost</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>227</td>\n",
              "      <td>1_thank_flight_plane_appreci</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2</td>\n",
              "      <td>153</td>\n",
              "      <td>2_cancel_flight_flightl_get</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3</td>\n",
              "      <td>135</td>\n",
              "      <td>3_custom_servic_care_thank</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>4</td>\n",
              "      <td>129</td>\n",
              "      <td>4_delay_flight_hour_time</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>5</td>\n",
              "      <td>103</td>\n",
              "      <td>5_gate_agent_wait_sit</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b44f59ad-737e-4ab2-8188-9fe67008580e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b44f59ad-737e-4ab2-8188-9fe67008580e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b44f59ad-737e-4ab2-8188-9fe67008580e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "   Topic  Count                          Name\n",
              "0 -1      2727   -1_flight_get_co_http       \n",
              "1  0      331    0_bag_baggag_check_lost     \n",
              "2  1      227    1_thank_flight_plane_appreci\n",
              "3  2      153    2_cancel_flight_flightl_get \n",
              "4  3      135    3_custom_servic_care_thank  \n",
              "5  4      129    4_delay_flight_hour_time    \n",
              "6  5      103    5_gate_agent_wait_sit       "
            ]
          },
          "execution_count": 644,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic_info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BQN-7mgn0ujI"
      },
      "outputs": [],
      "source": [
        "UAL_tweets['topics'] = UAL_topics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "id": "YwFXiKL7tVXF",
        "outputId": "cc1f1a78-4a66-439d-e274-2038771f26ea"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-3c72dce1-d2ab-4b11-be27-6d28cd3cc941\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>airline</th>\n",
              "      <th>name</th>\n",
              "      <th>text</th>\n",
              "      <th>tweet_created</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>topics</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>United</td>\n",
              "      <td>rdowning76</td>\n",
              "      <td>united thanks</td>\n",
              "      <td>2015-02-24 11:42:48</td>\n",
              "      <td>POSITIVE</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>United</td>\n",
              "      <td>CoreyAStewart</td>\n",
              "      <td>united thanks for taking care of that mr happy customer.</td>\n",
              "      <td>2015-02-24 11:42:41</td>\n",
              "      <td>POSITIVE</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>United</td>\n",
              "      <td>CoralReefer420</td>\n",
              "      <td>united still no refund or word via dm. please resolve this issue as your cancelled flightled flight was useless to my assistant s trip.</td>\n",
              "      <td>2015-02-24 11:39:45</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>United</td>\n",
              "      <td>lsalazarll</td>\n",
              "      <td>united delayed due to lack of crew and now delayed again because there s a long line for deicing... still need to improve service united</td>\n",
              "      <td>2015-02-24 11:39:25</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>United</td>\n",
              "      <td>rombaa</td>\n",
              "      <td>united thanks we filled it out. how s our luck with this is it common</td>\n",
              "      <td>2015-02-24 11:38:15</td>\n",
              "      <td>POSITIVE</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3c72dce1-d2ab-4b11-be27-6d28cd3cc941')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3c72dce1-d2ab-4b11-be27-6d28cd3cc941 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3c72dce1-d2ab-4b11-be27-6d28cd3cc941');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "   tweet_id airline            name  \\\n",
              "0  1         United  rdowning76       \n",
              "1  2         United  CoreyAStewart    \n",
              "2  3         United  CoralReefer420   \n",
              "3  4         United  lsalazarll       \n",
              "4  5         United  rombaa           \n",
              "\n",
              "                                                                                                                                       text  \\\n",
              "0  united thanks                                                                                                                              \n",
              "1  united thanks for taking care of that mr happy customer.                                                                                   \n",
              "2  united still no refund or word via dm. please resolve this issue as your cancelled flightled flight was useless to my assistant s trip.    \n",
              "3  united delayed due to lack of crew and now delayed again because there s a long line for deicing... still need to improve service united   \n",
              "4  united thanks we filled it out. how s our luck with this is it common                                                                      \n",
              "\n",
              "        tweet_created sentiment  topics  \n",
              "0 2015-02-24 11:42:48  POSITIVE  1       \n",
              "1 2015-02-24 11:42:41  POSITIVE  3       \n",
              "2 2015-02-24 11:39:45  NEGATIVE -1       \n",
              "3 2015-02-24 11:39:25  NEGATIVE -1       \n",
              "4 2015-02-24 11:38:15  POSITIVE -1       "
            ]
          },
          "execution_count": 646,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "UAL_tweets.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k6YHIUQSOLjF"
      },
      "source": [
        "**UAL Topics:**\n",
        "\n",
        "**Topic 0**\n",
        "Topic seems to be about lost baggage\n",
        "\n",
        "**Topic 1**\n",
        "Topic seems appreciation tweets/thank yous\n",
        "\n",
        "**Topic 2**\n",
        "Topic seems to be about cancellations and rebookings\n",
        "\n",
        "**Topic 3**\n",
        "Topic seems to be about contacting customer service\n",
        "\n",
        "**Topic 4**\n",
        "Topic seems to be about delays and misconnects\n",
        "\n",
        "**Topic 5**\n",
        "Topic seems to be about the gate agents, delays at the gate, and while taxiing\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F9A6ae_XYequ"
      },
      "source": [
        "### Sentiment analysis on each topic for United Airlines"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rlxfmkFaKyPN"
      },
      "source": [
        "This will be so that the client can understand how their customers feel about these main topics and where they need to improve\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 520
        },
        "id": "JkfD95-MMhDS",
        "outputId": "7263403a-97d1-4520-cf74-c02cbdbba49d"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-f62999d9-c1c8-4ea5-9887-206ac37a5e34\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>sentiment</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>topics</th>\n",
              "      <th>sentiment</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">-1</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>73.49</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>26.51</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>82.78</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>17.22</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>35.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>64.32</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>92.81</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>7.19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">3</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>78.52</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>21.48</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">4</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>90.70</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>9.30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">5</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>90.29</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>9.71</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f62999d9-c1c8-4ea5-9887-206ac37a5e34')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f62999d9-c1c8-4ea5-9887-206ac37a5e34 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f62999d9-c1c8-4ea5-9887-206ac37a5e34');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                  sentiment\n",
              "topics sentiment           \n",
              "-1     NEGATIVE   73.49    \n",
              "       POSITIVE   26.51    \n",
              " 0     NEGATIVE   82.78    \n",
              "       POSITIVE   17.22    \n",
              " 1     NEGATIVE   35.68    \n",
              "       POSITIVE   64.32    \n",
              " 2     NEGATIVE   92.81    \n",
              "       POSITIVE   7.19     \n",
              " 3     NEGATIVE   78.52    \n",
              "       POSITIVE   21.48    \n",
              " 4     NEGATIVE   90.70    \n",
              "       POSITIVE   9.30     \n",
              " 5     NEGATIVE   90.29    \n",
              "       POSITIVE   9.71     "
            ]
          },
          "execution_count": 647,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "ual_sentiments = UAL_tweets.groupby(['topics', 'sentiment']).agg({'sentiment': 'count'})\n",
        "topics = UAL_tweets.groupby(['topics']).agg({'sentiment': 'count'})\n",
        "ual_sentiment_pctg = round(ual_sentiments.div(topics, level='topics') * 100,2)\n",
        "ual_sentiment_pctg\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FnhkRvkHkJ5v"
      },
      "source": [
        "According to the sentiment analysis for United Airlines topic 1 (appreciation tweets) seems to have the most positive tweet rate (64%) and Topic 2 (cancellations and rebookings) seems to have the most negative responses - which inherently makes sense."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9RwHW7fykwom"
      },
      "source": [
        "### Topic detection for American Airlines"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "juGtJ5wBsTup"
      },
      "outputs": [],
      "source": [
        "AA_topics, AA_probabilities = topic_model.fit_transform(AA_data_clean)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "id": "6VZgxqbgsl9C",
        "outputId": "97896216-b82d-49db-f554-03bdd370f269"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-3ccafa66-d34e-42f8-ba6c-d18a74199045\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Topic</th>\n",
              "      <th>Count</th>\n",
              "      <th>Name</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1</td>\n",
              "      <td>1922</td>\n",
              "      <td>-1_flight_get_cancel_aa</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>186</td>\n",
              "      <td>0_call_hour_hold_wait</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>146</td>\n",
              "      <td>1_bag_baggag_check_still</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2</td>\n",
              "      <td>94</td>\n",
              "      <td>2_thank_appreci_much_done</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3</td>\n",
              "      <td>85</td>\n",
              "      <td>3_servic_custom_line_terribl</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>4</td>\n",
              "      <td>83</td>\n",
              "      <td>4_email_respons_tweet_twitter</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>5</td>\n",
              "      <td>78</td>\n",
              "      <td>5_cancel_flight_flightl_dfw</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3ccafa66-d34e-42f8-ba6c-d18a74199045')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3ccafa66-d34e-42f8-ba6c-d18a74199045 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3ccafa66-d34e-42f8-ba6c-d18a74199045');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "   Topic  Count                           Name\n",
              "0 -1      1922   -1_flight_get_cancel_aa      \n",
              "1  0      186    0_call_hour_hold_wait        \n",
              "2  1      146    1_bag_baggag_check_still     \n",
              "3  2      94     2_thank_appreci_much_done    \n",
              "4  3      85     3_servic_custom_line_terribl \n",
              "5  4      83     4_email_respons_tweet_twitter\n",
              "6  5      78     5_cancel_flight_flightl_dfw  "
            ]
          },
          "execution_count": 649,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic_info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 667
        },
        "id": "IoTHq8_Q2mVc",
        "outputId": "10d39ead-e880-4ed2-824e-ef05fdba0a0f"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-2.8.3.min.js\"></script>                <div id=\"661b1053-f6e6-4210-ba3d-fad5cbb8c18e\" class=\"plotly-graph-div\" style=\"height:650px; width:650px;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"661b1053-f6e6-4210-ba3d-fad5cbb8c18e\")) {                    Plotly.newPlot(                        \"661b1053-f6e6-4210-ba3d-fad5cbb8c18e\",                        [{\"customdata\":[[0,\"call | hour | hold | wait | back\",186],[1,\"bag | baggag | check | still | lost\",146],[2,\"thank | appreci | much | done | respons\",94],[3,\"servic | custom | line | terribl | call\",85],[4,\"email | respons | tweet | twitter | complaint\",83],[5,\"cancel | flight | flightl | dfw | tomorrow\",78]],\"hovertemplate\":\"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#B0BEC5\",\"size\":[186,146,94,85,83,78],\"sizemode\":\"area\",\"sizeref\":0.11625,\"symbol\":\"circle\",\"line\":{\"color\":\"DarkSlateGrey\",\"width\":2}},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[6.684407711029053,6.661930561065674,5.375500202178955,7.424851417541504,5.654151439666748,7.382953643798828],\"xaxis\":\"x\",\"y\":[5.405180931091309,6.231402397155762,6.402204513549805,5.6737589836120605,5.719875812530518,7.488711833953857],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"bar\":[{\"error_x\":{\"color\":\"rgb(36,36,36)\"},\"error_y\":{\"color\":\"rgb(36,36,36)\"},\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"rgb(36,36,36)\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"rgb(36,36,36)\"},\"baxis\":{\"endlinecolor\":\"rgb(36,36,36)\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"rgb(36,36,36)\"},\"type\":\"carpet\"}],\"choropleth\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"type\":\"choropleth\"}],\"contourcarpet\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"type\":\"contourcarpet\"}],\"contour\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"contour\"}],\"heatmapgl\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"heatmapgl\"}],\"heatmap\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"heatmap\"}],\"histogram2dcontour\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"histogram2dcontour\"}],\"histogram2d\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"histogram2d\"}],\"histogram\":[{\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.6}},\"type\":\"histogram\"}],\"mesh3d\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"type\":\"mesh3d\"}],\"parcoords\":[{\"line\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"parcoords\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}],\"scatter3d\":[{\"line\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatter3d\"}],\"scattercarpet\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattercarpet\"}],\"scattergeo\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattergeo\"}],\"scattergl\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattergl\"}],\"scattermapbox\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattermapbox\"}],\"scatterpolargl\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatterpolargl\"}],\"scatterpolar\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatterpolar\"}],\"scatter\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatter\"}],\"scatterternary\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatterternary\"}],\"surface\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"surface\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"rgb(237,237,237)\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"rgb(217,217,217)\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}]},\"layout\":{\"annotationdefaults\":{\"arrowhead\":0,\"arrowwidth\":1},\"autotypenumbers\":\"strict\",\"coloraxis\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"colorscale\":{\"diverging\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"sequential\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"sequentialminus\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]]},\"colorway\":[\"#1F77B4\",\"#FF7F0E\",\"#2CA02C\",\"#D62728\",\"#9467BD\",\"#8C564B\",\"#E377C2\",\"#7F7F7F\",\"#BCBD22\",\"#17BECF\"],\"font\":{\"color\":\"rgb(36,36,36)\"},\"geo\":{\"bgcolor\":\"white\",\"lakecolor\":\"white\",\"landcolor\":\"white\",\"showlakes\":true,\"showland\":true,\"subunitcolor\":\"white\"},\"hoverlabel\":{\"align\":\"left\"},\"hovermode\":\"closest\",\"mapbox\":{\"style\":\"light\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"white\",\"polar\":{\"angularaxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"},\"bgcolor\":\"white\",\"radialaxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"}},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"rgb(232,232,232)\",\"gridwidth\":2,\"linecolor\":\"rgb(36,36,36)\",\"showbackground\":true,\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"},\"yaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"rgb(232,232,232)\",\"gridwidth\":2,\"linecolor\":\"rgb(36,36,36)\",\"showbackground\":true,\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"},\"zaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"rgb(232,232,232)\",\"gridwidth\":2,\"linecolor\":\"rgb(36,36,36)\",\"showbackground\":true,\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"}},\"shapedefaults\":{\"fillcolor\":\"black\",\"line\":{\"width\":0},\"opacity\":0.3},\"ternary\":{\"aaxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"},\"baxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"},\"bgcolor\":\"white\",\"caxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"}},\"title\":{\"x\":0.05},\"xaxis\":{\"automargin\":true,\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"title\":{\"standoff\":15},\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"},\"yaxis\":{\"automargin\":true,\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"title\":{\"standoff\":15},\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"\"},\"visible\":false,\"range\":[4.569175171852112,8.53857913017273]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"\"},\"visible\":false,\"range\":[4.594403791427612,8.612018609046936]},\"legend\":{\"tracegroupgap\":0,\"itemsizing\":\"constant\"},\"margin\":{\"t\":60},\"title\":{\"font\":{\"size\":22,\"color\":\"Black\"},\"text\":\"<b>Intertopic Distance Map\",\"y\":0.95,\"x\":0.5,\"xanchor\":\"center\",\"yanchor\":\"top\"},\"hoverlabel\":{\"font\":{\"size\":16,\"family\":\"Rockwell\"},\"bgcolor\":\"white\"},\"width\":650,\"height\":650,\"sliders\":[{\"active\":0,\"pad\":{\"t\":50},\"steps\":[{\"args\":[{\"marker.color\":[[\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 0\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 1\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 2\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 3\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\"]]}],\"label\":\"Topic 4\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\"]]}],\"label\":\"Topic 5\",\"method\":\"update\"}]}],\"shapes\":[{\"line\":{\"color\":\"#CFD8DC\",\"width\":2},\"type\":\"line\",\"x0\":6.55387715101242,\"x1\":6.55387715101242,\"y0\":4.594403791427612,\"y1\":8.612018609046936},{\"line\":{\"color\":\"#9E9E9E\",\"width\":2},\"type\":\"line\",\"x0\":4.569175171852112,\"x1\":8.53857913017273,\"y0\":6.603211200237274,\"y1\":6.603211200237274}],\"annotations\":[{\"showarrow\":false,\"text\":\"D1\",\"x\":4.569175171852112,\"y\":6.603211200237274,\"yshift\":10},{\"showarrow\":false,\"text\":\"D2\",\"x\":6.55387715101242,\"xshift\":10,\"y\":8.612018609046936}]},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('661b1053-f6e6-4210-ba3d-fad5cbb8c18e');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "topic_model.visualize_topics()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i764ABvkQVmj",
        "outputId": "379fd520-ae99-446c-e308-befa7935130f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('call', 0.18283140929631417),\n",
              " ('hour', 0.10365192576399984),\n",
              " ('hold', 0.09152745360218029),\n",
              " ('wait', 0.08332983280609824),\n",
              " ('back', 0.07195495300471305),\n",
              " ('get', 0.06938841269357698),\n",
              " ('phone', 0.05951920730900851),\n",
              " ('system', 0.05653238801516709),\n",
              " ('hung', 0.05512651879474071),\n",
              " ('agent', 0.05503946994839271)]"
            ]
          },
          "execution_count": 651,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Understanding what is within each topic\n",
        "topic_model.get_topic(0)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CX_Ke7ZUQVmk"
      },
      "source": [
        "**Topic 0**\n",
        "\n",
        "Topic seems to be about call wait times"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FnGHrSWUQVmk",
        "outputId": "d13906ea-7902-465b-df2a-10c2b40bfd9e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('bag', 0.2638596350049143),\n",
              " ('baggag', 0.08570045542461342),\n",
              " ('check', 0.04335016337245549),\n",
              " ('still', 0.042398065098267074),\n",
              " ('lost', 0.040605495551950505),\n",
              " ('servic', 0.03343508341182018),\n",
              " ('claim', 0.033379875490339946),\n",
              " ('load', 0.031474012915076524),\n",
              " ('would', 0.029171454837321927),\n",
              " ('day', 0.028015006582239238)]"
            ]
          },
          "execution_count": 652,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XjGarek7QVmk"
      },
      "source": [
        "**Topic 1**\n",
        "\n",
        "Topic seems to be about lost baggage"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FeWqToz2QVmk",
        "outputId": "e00dc0bf-a778-492f-9653-aa50990136be"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('thank', 0.6574185567768543),\n",
              " ('appreci', 0.21271860609371762),\n",
              " ('much', 0.19144674548434587),\n",
              " ('done', 0.13834989298492853),\n",
              " ('respons', 0.12828858283886438),\n",
              " ('great', 0.12058257878152824),\n",
              " ('help', 0.11107430036984108),\n",
              " ('ye', 0.07997948648872376),\n",
              " ('awesom', 0.07505199072682019),\n",
              " ('ok', 0.07332971178802444)]"
            ]
          },
          "execution_count": 653,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(2)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vnszra8WQVmk"
      },
      "source": [
        "**Topic 2**\n",
        "\n",
        "Topic seems to be appreciation tweets/thank yous"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mG5Y1j-LQVmk",
        "outputId": "efec8d7a-8874-45da-e378-d77ffd50e990"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('servic', 0.2923519597400248),\n",
              " ('custom', 0.2787286964152514),\n",
              " ('line', 0.049322917166332025),\n",
              " ('terribl', 0.03686882243332842),\n",
              " ('call', 0.03633377559976639),\n",
              " ('flight', 0.031924975045829945),\n",
              " ('cancel', 0.03013191978381585),\n",
              " ('realli', 0.029971863553757045),\n",
              " ('poor', 0.02992392455685563),\n",
              " ('inform', 0.02619426133931641)]"
            ]
          },
          "execution_count": 654,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(3)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HIo5sxFdQVmk"
      },
      "source": [
        "**Topic 3**\n",
        "\n",
        "Topic seems to be about negatie experiences with customer service"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tSqhKni3QVmk",
        "outputId": "c82ce52d-55ce-4688-8c31-a4bc2dfbf522"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('email', 0.12655647612265322),\n",
              " ('respons', 0.09419082792642937),\n",
              " ('tweet', 0.09383762130728279),\n",
              " ('twitter', 0.08883689365620275),\n",
              " ('complaint', 0.07677106229737471),\n",
              " ('media', 0.07058571930788843),\n",
              " ('custom', 0.05772803697407813),\n",
              " ('relat', 0.05646857544631074),\n",
              " ('send', 0.046918810653641396),\n",
              " ('respond', 0.04608629110616369)]"
            ]
          },
          "execution_count": 655,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(4)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p_49gGLEQVmk"
      },
      "source": [
        "**Topic 4**\n",
        "\n",
        "Topic seems to be about contacting customer service via multiple modes of communication"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jMgmLws3QVmk",
        "outputId": "f5851296-bfac-485d-b7c2-3b8a0bb68905"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('cancel', 0.2795141017793037),\n",
              " ('flight', 0.2138962688185572),\n",
              " ('flightl', 0.2029734786282398),\n",
              " ('dfw', 0.10244853601469496),\n",
              " ('tomorrow', 0.08404988979315309),\n",
              " ('dalla', 0.05076766380330695),\n",
              " ('hour', 0.0498699980944402),\n",
              " ('help', 0.03793377900575309),\n",
              " ('one', 0.03784898220764171),\n",
              " ('flightlat', 0.03716229764221983)]"
            ]
          },
          "execution_count": 656,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(5)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WJzMo9ySQVmk"
      },
      "source": [
        "**Topic 5**\n",
        "\n",
        "Topic seems to be about cancellations of flights"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GRDtPhtyRc4f"
      },
      "source": [
        "**American Airlines Topics:**\n",
        "\n",
        "**Topic 0**\n",
        "Topic seems to be about call wait times\n",
        " \n",
        "**Topic 1**\n",
        "Topic seems to be about lost baggage\n",
        " \n",
        "**Topic 2**\n",
        "Topic seems to be appreciation tweets/thank yous\n",
        " \n",
        "**Topic 3**\n",
        "Topic seems to be about negatie experiences with customer service\n",
        "\n",
        "**Topic 4**\n",
        "Topic seems to be about contacting customer service via multiple modes of communication\n",
        " \n",
        "**Topic 5**\n",
        "Topic seems to be about cancellations of flights\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2ukdKUeyQVmk",
        "outputId": "0a1d1a95-ca38-4e25-d71c-907ed674b740"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{3: ['custom servic disgrac ',\n",
              "  'great custom servic thank ',\n",
              "  'allow convers upset custom custom servic ',\n",
              "  'cancel flightl flight mainten custom servic desk empti get hotel least http co ypbkcirbxu ',\n",
              "  'request call back cancel flight flight dfw take long rerout custom servic lack ',\n",
              "  'know mean custom servic nightmar paid direct flight bump dalla flight cancel flight terribl '],\n",
              " 2: ['thank ',\n",
              "  'thank ',\n",
              "  'ye thank ',\n",
              "  '',\n",
              "  'understand ',\n",
              "  'tick ',\n",
              "  'thank respons much appreci ',\n",
              "  'thank much appreci ',\n",
              "  'appreci '],\n",
              " 0: ['one get hold get past autom repli hang frustrat ',\n",
              "  'hold estim hr hold ',\n",
              "  'fine would call left messag told would hour call heard anyth yet ',\n",
              "  'yet receiv assist one agent secur new connect mani miss work tomorrow ',\n",
              "  'will hold agent help keep hang hiremorepoepl ',\n",
              "  'wait time hour talk exec plt agent go ',\n",
              "  'flight cancel flightl call said call back hung say wait hr ',\n",
              "  'let get line call back hour hope ticket left flight want still late flightr ',\n",
              "  'hr late flightr get call back immedi disconnect '],\n",
              " 4: ['share mani mani other filmcrew media wsj nytim latim chicagotribun ',\n",
              "  'care custom break gear tell anyth wsj latim chicagotribun media filmcrew ',\n",
              "  'cnn nbsnewsr cnnbrk except work youv done noth help ',\n",
              "  'thank contact american email address written unmonitor account ',\n",
              "  'told would email receiv anyth yet follow ',\n",
              "  'fill whole form receiv poor poor respons direct email address repli ',\n",
              "  'thank respons twitter truli impress ',\n",
              "  'go keep tweet come till aa someth know noth hope go way like ',\n",
              "  'manuel yeah tweet platitud think twitter bot care '],\n",
              " 5: ['respons flight montros cancel flightl ',\n",
              "  'cancel flightl flight buffalo weather lie us said airlin cancel flightl ',\n",
              "  'jump gun cancel flight flight could made snow cancel flightlat ridicul ',\n",
              "  'best way check delay cancel flightlat flight dfw tuesday ',\n",
              "  'expect flight dfw cancel flightl tomorrow afternoon ',\n",
              "  'flight vega dfw cancel flightl sunday monday whammi today '],\n",
              " 1: ['yvonneokaka get person respons apolog crew forgotten load baggag onto flight ',\n",
              "  'actingoutmgmnt make sure rememb load bag onto plane cancel flight flight forget ',\n",
              "  'misplac bag hr ago tell lost stuff ']}"
            ]
          },
          "execution_count": 657,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#the most representative tweets within each topic:\n",
        "topic_model.get_representative_docs()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6hvkvngKs4o2"
      },
      "outputs": [],
      "source": [
        "AA_tweets['topics'] = AA_topics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "id": "X1Lvc0c2k7Kc",
        "outputId": "18714705-56e3-4649-a114-662c64aeb171"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-74630188-b136-4342-a431-31840f834d4a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>airline</th>\n",
              "      <th>name</th>\n",
              "      <th>text</th>\n",
              "      <th>tweet_created</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>topics</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>6242</th>\n",
              "      <td>6243</td>\n",
              "      <td>American</td>\n",
              "      <td>SweeLoTmac</td>\n",
              "      <td>americanair why would i even consider continuing your point program when i received no perks or continued bad customer service senseless</td>\n",
              "      <td>2015-02-24 11:44:31</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6243</th>\n",
              "      <td>6244</td>\n",
              "      <td>American</td>\n",
              "      <td>LancasterPattie</td>\n",
              "      <td>americanair we ve already made other arrangements ourselves.</td>\n",
              "      <td>2015-02-24 11:43:32</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6244</th>\n",
              "      <td>6245</td>\n",
              "      <td>American</td>\n",
              "      <td>ELLLORRAC</td>\n",
              "      <td>americanair thanks for getting back to me. but i will find other airlines in the future.</td>\n",
              "      <td>2015-02-24 11:43:05</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6245</th>\n",
              "      <td>6246</td>\n",
              "      <td>American</td>\n",
              "      <td>SweeLoTmac</td>\n",
              "      <td>americanair why would i pay to reactivate my points that are only useful for certain flights that aren t even worth</td>\n",
              "      <td>2015-02-24 11:43:05</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6246</th>\n",
              "      <td>6247</td>\n",
              "      <td>American</td>\n",
              "      <td>LauraMolito</td>\n",
              "      <td>americanair stranded for hours in mia patrick casimir has been the only aa staff to apologize for the great inconvenience unreal</td>\n",
              "      <td>2015-02-24 11:41:02</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-74630188-b136-4342-a431-31840f834d4a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-74630188-b136-4342-a431-31840f834d4a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-74630188-b136-4342-a431-31840f834d4a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "      tweet_id   airline             name  \\\n",
              "6242  6243      American  SweeLoTmac        \n",
              "6243  6244      American  LancasterPattie   \n",
              "6244  6245      American  ELLLORRAC         \n",
              "6245  6246      American  SweeLoTmac        \n",
              "6246  6247      American  LauraMolito       \n",
              "\n",
              "                                                                                                                                          text  \\\n",
              "6242  americanair why would i even consider continuing your point program when i received no perks or continued bad customer service senseless   \n",
              "6243  americanair we ve already made other arrangements ourselves.                                                                               \n",
              "6244  americanair thanks for getting back to me. but i will find other airlines in the future.                                                   \n",
              "6245  americanair why would i pay to reactivate my points that are only useful for certain flights that aren t even worth                        \n",
              "6246  americanair stranded for hours in mia patrick casimir has been the only aa staff to apologize for the great inconvenience unreal           \n",
              "\n",
              "           tweet_created sentiment  topics  \n",
              "6242 2015-02-24 11:44:31  NEGATIVE  3       \n",
              "6243 2015-02-24 11:43:32  NEGATIVE -1       \n",
              "6244 2015-02-24 11:43:05  NEGATIVE -1       \n",
              "6245 2015-02-24 11:43:05  NEGATIVE -1       \n",
              "6246 2015-02-24 11:41:02  NEGATIVE -1       "
            ]
          },
          "execution_count": 659,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "AA_tweets.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_kZJQzhSOfXv"
      },
      "source": [
        "### Sentiment analysis on each topic for American Airlines"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 520
        },
        "id": "wiwDn9QpQjPL",
        "outputId": "2fee7df6-8752-4e12-cd55-8ae8ba50f80b"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-b96afccc-747b-4206-8eba-1e0a8616170d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>sentiment</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>topics</th>\n",
              "      <th>sentiment</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">-1</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>77.42</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>22.58</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>91.40</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>8.60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>90.41</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>9.59</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>14.89</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>85.11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">3</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>83.53</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>16.47</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">4</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>73.49</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>26.51</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">5</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>94.87</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>5.13</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b96afccc-747b-4206-8eba-1e0a8616170d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b96afccc-747b-4206-8eba-1e0a8616170d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b96afccc-747b-4206-8eba-1e0a8616170d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                  sentiment\n",
              "topics sentiment           \n",
              "-1     NEGATIVE   77.42    \n",
              "       POSITIVE   22.58    \n",
              " 0     NEGATIVE   91.40    \n",
              "       POSITIVE   8.60     \n",
              " 1     NEGATIVE   90.41    \n",
              "       POSITIVE   9.59     \n",
              " 2     NEGATIVE   14.89    \n",
              "       POSITIVE   85.11    \n",
              " 3     NEGATIVE   83.53    \n",
              "       POSITIVE   16.47    \n",
              " 4     NEGATIVE   73.49    \n",
              "       POSITIVE   26.51    \n",
              " 5     NEGATIVE   94.87    \n",
              "       POSITIVE   5.13     "
            ]
          },
          "execution_count": 660,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "aa_sentiments = AA_tweets.groupby(['topics', 'sentiment']).agg({'sentiment': 'count'})\n",
        "topics = AA_tweets.groupby(['topics']).agg({'sentiment': 'count'})\n",
        "aa_sentiment_pctg = round(aa_sentiments.div(topics, level='topics') * 100,2)\n",
        "aa_sentiment_pctg\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mN8jCKdZSdLb"
      },
      "source": [
        "Most positive: Topic 2 (appreciation tweets) and the most negative is Topic 5 - flight cancellations"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KKxQwdlYlOb2"
      },
      "source": [
        "### Topic detection for Southwest"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NX7TRo9nT07e"
      },
      "outputs": [],
      "source": [
        "SW_topics, SW_probabilities = topic_model.fit_transform(SW_data_clean)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "id": "FJw3oOHbT07e",
        "outputId": "3cdc8936-7ac4-403e-d3f4-31ce5f0ad9e1"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-f922e869-21d8-4fac-8771-e2552e9349b2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Topic</th>\n",
              "      <th>Count</th>\n",
              "      <th>Name</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1</td>\n",
              "      <td>1543</td>\n",
              "      <td>-1_flight_co_http_fli</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>226</td>\n",
              "      <td>0_thank_dm_follow_sent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>224</td>\n",
              "      <td>1_cancel_flight_flightl_hold</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2</td>\n",
              "      <td>118</td>\n",
              "      <td>2_hold_hour_call_phone</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3</td>\n",
              "      <td>114</td>\n",
              "      <td>3_ticket_imaginedragon_show_would</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>4</td>\n",
              "      <td>99</td>\n",
              "      <td>4_bag_baggag_hour_lost</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>5</td>\n",
              "      <td>84</td>\n",
              "      <td>5_custom_servic_call_terribl</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f922e869-21d8-4fac-8771-e2552e9349b2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f922e869-21d8-4fac-8771-e2552e9349b2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f922e869-21d8-4fac-8771-e2552e9349b2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "   Topic  Count                               Name\n",
              "0 -1      1543   -1_flight_co_http_fli            \n",
              "1  0      226    0_thank_dm_follow_sent           \n",
              "2  1      224    1_cancel_flight_flightl_hold     \n",
              "3  2      118    2_hold_hour_call_phone           \n",
              "4  3      114    3_ticket_imaginedragon_show_would\n",
              "5  4      99     4_bag_baggag_hour_lost           \n",
              "6  5      84     5_custom_servic_call_terribl     "
            ]
          },
          "execution_count": 662,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic_info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 667
        },
        "id": "VMXpoBXhT07e",
        "outputId": "94243348-8e41-4176-98c1-3d5f13163b4a"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-2.8.3.min.js\"></script>                <div id=\"2ebaf172-326a-4f75-8fd5-50d2aac5b852\" class=\"plotly-graph-div\" style=\"height:650px; width:650px;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"2ebaf172-326a-4f75-8fd5-50d2aac5b852\")) {                    Plotly.newPlot(                        \"2ebaf172-326a-4f75-8fd5-50d2aac5b852\",                        [{\"customdata\":[[0,\"thank | dm | follow | sent | pleas\",226],[1,\"cancel | flight | flightl | hold | get\",224],[2,\"hold | hour | call | phone | minut\",118],[3,\"ticket | imaginedragon | show | would | dragon\",114],[4,\"bag | baggag | hour | lost | wait\",99],[5,\"custom | servic | call | terribl | hold\",84]],\"hovertemplate\":\"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#B0BEC5\",\"size\":[226,224,118,114,99,84],\"sizemode\":\"area\",\"sizeref\":0.14125,\"symbol\":\"circle\",\"line\":{\"color\":\"DarkSlateGrey\",\"width\":2}},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[2.4868831634521484,3.9638993740081787,3.2541873455047607,1.926759123802185,4.0577073097229,4.534849166870117],\"xaxis\":\"x\",\"y\":[4.509650707244873,4.112471580505371,4.358396530151367,4.279784679412842,3.4547057151794434,4.328311443328857],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"bar\":[{\"error_x\":{\"color\":\"rgb(36,36,36)\"},\"error_y\":{\"color\":\"rgb(36,36,36)\"},\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"rgb(36,36,36)\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"rgb(36,36,36)\"},\"baxis\":{\"endlinecolor\":\"rgb(36,36,36)\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"rgb(36,36,36)\"},\"type\":\"carpet\"}],\"choropleth\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"type\":\"choropleth\"}],\"contourcarpet\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"type\":\"contourcarpet\"}],\"contour\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"contour\"}],\"heatmapgl\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"heatmapgl\"}],\"heatmap\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"heatmap\"}],\"histogram2dcontour\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"histogram2dcontour\"}],\"histogram2d\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"histogram2d\"}],\"histogram\":[{\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.6}},\"type\":\"histogram\"}],\"mesh3d\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"type\":\"mesh3d\"}],\"parcoords\":[{\"line\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"parcoords\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}],\"scatter3d\":[{\"line\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatter3d\"}],\"scattercarpet\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattercarpet\"}],\"scattergeo\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattergeo\"}],\"scattergl\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattergl\"}],\"scattermapbox\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattermapbox\"}],\"scatterpolargl\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatterpolargl\"}],\"scatterpolar\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatterpolar\"}],\"scatter\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatter\"}],\"scatterternary\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatterternary\"}],\"surface\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"surface\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"rgb(237,237,237)\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"rgb(217,217,217)\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}]},\"layout\":{\"annotationdefaults\":{\"arrowhead\":0,\"arrowwidth\":1},\"autotypenumbers\":\"strict\",\"coloraxis\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"colorscale\":{\"diverging\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"sequential\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"sequentialminus\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]]},\"colorway\":[\"#1F77B4\",\"#FF7F0E\",\"#2CA02C\",\"#D62728\",\"#9467BD\",\"#8C564B\",\"#E377C2\",\"#7F7F7F\",\"#BCBD22\",\"#17BECF\"],\"font\":{\"color\":\"rgb(36,36,36)\"},\"geo\":{\"bgcolor\":\"white\",\"lakecolor\":\"white\",\"landcolor\":\"white\",\"showlakes\":true,\"showland\":true,\"subunitcolor\":\"white\"},\"hoverlabel\":{\"align\":\"left\"},\"hovermode\":\"closest\",\"mapbox\":{\"style\":\"light\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"white\",\"polar\":{\"angularaxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"},\"bgcolor\":\"white\",\"radialaxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"}},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"rgb(232,232,232)\",\"gridwidth\":2,\"linecolor\":\"rgb(36,36,36)\",\"showbackground\":true,\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"},\"yaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"rgb(232,232,232)\",\"gridwidth\":2,\"linecolor\":\"rgb(36,36,36)\",\"showbackground\":true,\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"},\"zaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"rgb(232,232,232)\",\"gridwidth\":2,\"linecolor\":\"rgb(36,36,36)\",\"showbackground\":true,\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"}},\"shapedefaults\":{\"fillcolor\":\"black\",\"line\":{\"width\":0},\"opacity\":0.3},\"ternary\":{\"aaxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"},\"baxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"},\"bgcolor\":\"white\",\"caxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"}},\"title\":{\"x\":0.05},\"xaxis\":{\"automargin\":true,\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"title\":{\"standoff\":15},\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"},\"yaxis\":{\"automargin\":true,\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"title\":{\"standoff\":15},\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"\"},\"visible\":false,\"range\":[1.6377452552318572,5.215076541900634]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"\"},\"visible\":false,\"range\":[2.9364998579025268,5.186098313331604]},\"legend\":{\"tracegroupgap\":0,\"itemsizing\":\"constant\"},\"margin\":{\"t\":60},\"title\":{\"font\":{\"size\":22,\"color\":\"Black\"},\"text\":\"<b>Intertopic Distance Map\",\"y\":0.95,\"x\":0.5,\"xanchor\":\"center\",\"yanchor\":\"top\"},\"hoverlabel\":{\"font\":{\"size\":16,\"family\":\"Rockwell\"},\"bgcolor\":\"white\"},\"width\":650,\"height\":650,\"sliders\":[{\"active\":0,\"pad\":{\"t\":50},\"steps\":[{\"args\":[{\"marker.color\":[[\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 0\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 1\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 2\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 3\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\"]]}],\"label\":\"Topic 4\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\"]]}],\"label\":\"Topic 5\",\"method\":\"update\"}]}],\"shapes\":[{\"line\":{\"color\":\"#CFD8DC\",\"width\":2},\"type\":\"line\",\"x0\":3.426410898566246,\"x1\":3.426410898566246,\"y0\":2.9364998579025268,\"y1\":5.186098313331604},{\"line\":{\"color\":\"#9E9E9E\",\"width\":2},\"type\":\"line\",\"x0\":1.6377452552318572,\"x1\":5.215076541900634,\"y0\":4.061299085617065,\"y1\":4.061299085617065}],\"annotations\":[{\"showarrow\":false,\"text\":\"D1\",\"x\":1.6377452552318572,\"y\":4.061299085617065,\"yshift\":10},{\"showarrow\":false,\"text\":\"D2\",\"x\":3.426410898566246,\"xshift\":10,\"y\":5.186098313331604}]},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('2ebaf172-326a-4f75-8fd5-50d2aac5b852');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "topic_model.visualize_topics()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ybb_D-lIT07e",
        "outputId": "b4694956-13ec-44c0-96f5-c2a98bb34315"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('thank', 0.3158023115635206),\n",
              " ('dm', 0.28520362191565063),\n",
              " ('follow', 0.23253597145533253),\n",
              " ('sent', 0.14993822558559658),\n",
              " ('pleas', 0.08872834708223198),\n",
              " ('send', 0.06486664436152857),\n",
              " ('info', 0.06334457319807864),\n",
              " ('let', 0.061140794329567066),\n",
              " ('got', 0.056018156351170585),\n",
              " ('guy', 0.0518964505741713)]"
            ]
          },
          "execution_count": 664,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Understanding what is within each topic\n",
        "topic_model.get_topic(0)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eKtSMQ2ST07e"
      },
      "source": [
        "**Topic 0**\n",
        "\n",
        "Topic seems to be about Twitter specific actions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "As3fexvBT07f",
        "outputId": "d8550c41-39a7-48d2-fad4-3fe12b731fcb"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('cancel', 0.2122250334489206),\n",
              " ('flight', 0.1844137726450122),\n",
              " ('flightl', 0.13175371442150313),\n",
              " ('hold', 0.06670825215896148),\n",
              " ('get', 0.04435852266314852),\n",
              " ('rebook', 0.04404724447355021),\n",
              " ('help', 0.039353648847563316),\n",
              " ('refund', 0.03909430688328662),\n",
              " ('hour', 0.03604316153928753),\n",
              " ('flightlat', 0.03359065579210794)]"
            ]
          },
          "execution_count": 665,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JOzeFD5wT07f"
      },
      "source": [
        "**Topic 1**\n",
        "\n",
        "Topic seems to be about cancellations, rebookings and refunds"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xjgZpYWpT07f",
        "outputId": "e8d67854-b9d9-45c6-8a1d-d0e4b26a99f8"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('hold', 0.17593765189740884),\n",
              " ('hour', 0.13758820376401734),\n",
              " ('call', 0.12864681201738115),\n",
              " ('phone', 0.12266393130080698),\n",
              " ('minut', 0.09984633741371561),\n",
              " ('min', 0.0634950574307513),\n",
              " ('wait', 0.058715609934854404),\n",
              " ('time', 0.052879767064417466),\n",
              " ('answer', 0.05141282007619317),\n",
              " ('help', 0.042487987972122054)]"
            ]
          },
          "execution_count": 666,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(2)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "00psQCEXT07f"
      },
      "source": [
        "**Topic 2**\n",
        "\n",
        "Topic seems to be about customer service and wait times for call centers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L898BGkuT07f",
        "outputId": "2968f685-e370-4215-b26e-e10f57d09711"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('ticket', 0.17178524836876385),\n",
              " ('imaginedragon', 0.1535548885827457),\n",
              " ('show', 0.12890628515053734),\n",
              " ('would', 0.11785199787077236),\n",
              " ('dragon', 0.11376552919325979),\n",
              " ('vega', 0.10400007262237879),\n",
              " ('love', 0.09934302319705868),\n",
              " ('imagin', 0.08500575167034885),\n",
              " ('atlanta', 0.0707884442490527),\n",
              " ('see', 0.06777127403296207)]"
            ]
          },
          "execution_count": 667,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(3)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "52jmePsjT07f"
      },
      "source": [
        "**Topic 3**\n",
        "\n",
        "Topic seems to be about Imagine Dragons (they performed on a SW flight in feb 2021!!)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tsKxP9l_T07f",
        "outputId": "5781596e-f89a-4b99-aeee-c041bf7ce1a6"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('bag', 0.2928355343049211),\n",
              " ('baggag', 0.08388217467338807),\n",
              " ('hour', 0.05934614767651838),\n",
              " ('lost', 0.05427670125925112),\n",
              " ('wait', 0.05248459793124999),\n",
              " ('cloth', 0.04917743144189519),\n",
              " ('deliv', 0.0430302525116583),\n",
              " ('get', 0.04291073063623609),\n",
              " ('late', 0.03429209587247683),\n",
              " ('claim', 0.031302374364501236)]"
            ]
          },
          "execution_count": 668,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(4)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h8YbWzyvT07f"
      },
      "source": [
        "**Topic 4**\n",
        "\n",
        "Topic seems to be about lost baggage"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V9IurWfJT07f",
        "outputId": "e4a44ab2-2414-4f89-da9d-9b870245d1d6"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('custom', 0.2810876676035977),\n",
              " ('servic', 0.2735919964675018),\n",
              " ('call', 0.05701529637648425),\n",
              " ('terribl', 0.05695391362443524),\n",
              " ('hold', 0.05366648081776917),\n",
              " ('disappoint', 0.04649469796454115),\n",
              " ('terri', 0.045590593221087355),\n",
              " ('hour', 0.042528283627751746),\n",
              " ('help', 0.04221310502843111),\n",
              " ('sw', 0.04016952491428424)]"
            ]
          },
          "execution_count": 669,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic(5)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fNYY6n5MT07f"
      },
      "source": [
        "**Topic 5**\n",
        "\n",
        "Topic seems to be about bad experiences with customer service"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mQvlXytwT07g"
      },
      "source": [
        "**SW Topics**\n",
        "\n",
        "\n",
        "**Topic 0**\n",
        "Topic seems to be about Twitter specific actions\n",
        " \n",
        "**Topic 1**\n",
        "Topic seems to be about cancellations, rebookings and refunds\n",
        " \n",
        "**Topic 2**\n",
        "Topic seems to be about customer service and wait times for call centers\n",
        " \n",
        "**Topic 3**\n",
        "Topic seems to be about Imagine Dragons  (they performed on a SW flight in feb 2021 and SW did a ticket give away)\n",
        " \n",
        "**Topic 4**\n",
        "Topic seems to be about lost baggage\n",
        " \n",
        "**Topic 5**\n",
        "Topic seems to be about bad experiences with customer service\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V0wA8XtrT07f",
        "outputId": "21f35676-2c27-4cd5-85bb-604c02d5d8f2"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{2: ['ask lindsey call instead dm like schoolgirl far call get tire type littl note ',\n",
              "  'lukewyckoff name jane want call ',\n",
              "  'hold hour cool guy '],\n",
              " 0: ['time thank ', 'thank ', 'awesom thank ', 'zz ', 'dm sent thank ', ''],\n",
              " 1: ['way receiv refund trip cancel flight onlin instead call phone line super busi ',\n",
              "  'son flight cancel flightl today hold hour avail help rebook ',\n",
              "  'hold hour reschedul cancel flightl flight morn give need help '],\n",
              " 3: ['guy gonna give away ticket imaginedragon vinylvega ',\n",
              "  'show jacksonvil want ticketsforjax ',\n",
              "  'oh atlanta ',\n",
              "  'love dragonss lauren holi sht ',\n",
              "  'love dragonss holi fuckinf shit ',\n",
              "  'make dream come true scaveng hunt win chanc see imagin dragon go ',\n",
              "  'get leatheri heat even get vega ',\n",
              "  'pleas help would die see vega show would amaz hear song real ',\n",
              "  'tell fam vega '],\n",
              " 4: ['cool bag take bit longer dont give baggag blue ball turn carousel tell come ',\n",
              "  'hey rememb time lost husband bag nowarmcloth ',\n",
              "  'min get bag check atx miss flight spend day bwi get baggag end epitimeoffail '],\n",
              " 5: ['thank take care today michel rock custom servic gate hou ',\n",
              "  'thank great custom servic today ',\n",
              "  'thank great custom servic far accid happen understand hope everyth work ',\n",
              "  'disappoint custom servic right flight one cancel flightl airport amp lose day trip ',\n",
              "  'flight cancel flightl one week custom servic ',\n",
              "  'get run around today custom servic amp disconnect custom relat given twice help ',\n",
              "  'hour hold custom servic never us sw ',\n",
              "  'hold hour speak custom servic rep heck southwest customerservic ',\n",
              "  'hold two hour final hung abl need without custom servic eventu ']}"
            ]
          },
          "execution_count": 670,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#the most representative tweets within each topic:\n",
        "topic_model.get_representative_docs()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HqPfzrWfT07f"
      },
      "outputs": [],
      "source": [
        "SW_tweets['topics'] = SW_topics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 581
        },
        "id": "2YSUVJ0bwuZN",
        "outputId": "65c82c73-5523-4270-c873-25d434794c5f"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-f519c216-c6ce-4624-9711-a2414c4fce3f\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>airline</th>\n",
              "      <th>name</th>\n",
              "      <th>text</th>\n",
              "      <th>tweet_created</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>topics</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3822</th>\n",
              "      <td>3823</td>\n",
              "      <td>Southwest</td>\n",
              "      <td>thisradlove</td>\n",
              "      <td>southwestair still waiting. just hit one hour.</td>\n",
              "      <td>2015-02-24 11:47:53</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3831</th>\n",
              "      <td>3832</td>\n",
              "      <td>Southwest</td>\n",
              "      <td>tylerhansen331</td>\n",
              "      <td>southwestair thank you so much completely made things right</td>\n",
              "      <td>2015-02-24 11:25:00</td>\n",
              "      <td>POSITIVE</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3833</th>\n",
              "      <td>3834</td>\n",
              "      <td>Southwest</td>\n",
              "      <td>brittanylinnes</td>\n",
              "      <td>southwestair can you follow me so i can send you the info</td>\n",
              "      <td>2015-02-24 11:21:16</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3839</th>\n",
              "      <td>3840</td>\n",
              "      <td>Southwest</td>\n",
              "      <td>NardosKing2</td>\n",
              "      <td>southwestair i have been on hold for over minutes please help http t.co fx bijlxat</td>\n",
              "      <td>2015-02-24 11:05:00</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3844</th>\n",
              "      <td>3845</td>\n",
              "      <td>Southwest</td>\n",
              "      <td>Sarahtherower</td>\n",
              "      <td>southwestair can you have someone call me back i have been on hold two times today for over min and still haven t gotten through</td>\n",
              "      <td>2015-02-24 10:46:19</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3850</th>\n",
              "      <td>3851</td>\n",
              "      <td>Southwest</td>\n",
              "      <td>Marc_Hoffman</td>\n",
              "      <td>southwestair do you promise to not unfollow me</td>\n",
              "      <td>2015-02-24 10:43:21</td>\n",
              "      <td>POSITIVE</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3851</th>\n",
              "      <td>3852</td>\n",
              "      <td>Southwest</td>\n",
              "      <td>NecroticDoctor</td>\n",
              "      <td>southwestair just dmed you my confirmation number.</td>\n",
              "      <td>2015-02-24 10:36:36</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3855</th>\n",
              "      <td>3856</td>\n",
              "      <td>Southwest</td>\n",
              "      <td>unknownfilmaker</td>\n",
              "      <td>southwestair oh is it only in atlanta</td>\n",
              "      <td>2015-02-24 10:27:09</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3858</th>\n",
              "      <td>3859</td>\n",
              "      <td>Southwest</td>\n",
              "      <td>jparkermastin</td>\n",
              "      <td>southwestair got my bags hours late flightr and an offer of to go pick them up. taxi costs one way. spent for the inconvenience.</td>\n",
              "      <td>2015-02-24 10:22:26</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3860</th>\n",
              "      <td>3861</td>\n",
              "      <td>Southwest</td>\n",
              "      <td>briman007</td>\n",
              "      <td>southwestair sure please follow me so i can do so</td>\n",
              "      <td>2015-02-24 10:20:47</td>\n",
              "      <td>POSITIVE</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f519c216-c6ce-4624-9711-a2414c4fce3f')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f519c216-c6ce-4624-9711-a2414c4fce3f button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f519c216-c6ce-4624-9711-a2414c4fce3f');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "      tweet_id    airline             name  \\\n",
              "3822  3823      Southwest  thisradlove       \n",
              "3831  3832      Southwest  tylerhansen331    \n",
              "3833  3834      Southwest  brittanylinnes    \n",
              "3839  3840      Southwest  NardosKing2       \n",
              "3844  3845      Southwest  Sarahtherower     \n",
              "3850  3851      Southwest  Marc_Hoffman      \n",
              "3851  3852      Southwest  NecroticDoctor    \n",
              "3855  3856      Southwest  unknownfilmaker   \n",
              "3858  3859      Southwest  jparkermastin     \n",
              "3860  3861      Southwest  briman007         \n",
              "\n",
              "                                                                                                                                  text  \\\n",
              "3822  southwestair still waiting. just hit one hour.                                                                                     \n",
              "3831  southwestair thank you so much completely made things right                                                                        \n",
              "3833  southwestair can you follow me so i can send you the info                                                                          \n",
              "3839  southwestair i have been on hold for over minutes please help http t.co fx bijlxat                                                 \n",
              "3844  southwestair can you have someone call me back i have been on hold two times today for over min and still haven t gotten through   \n",
              "3850  southwestair do you promise to not unfollow me                                                                                     \n",
              "3851  southwestair just dmed you my confirmation number.                                                                                 \n",
              "3855  southwestair oh is it only in atlanta                                                                                              \n",
              "3858  southwestair got my bags hours late flightr and an offer of to go pick them up. taxi costs one way. spent for the inconvenience.   \n",
              "3860  southwestair sure please follow me so i can do so                                                                                  \n",
              "\n",
              "           tweet_created sentiment  topics  \n",
              "3822 2015-02-24 11:47:53  NEGATIVE  2       \n",
              "3831 2015-02-24 11:25:00  POSITIVE  0       \n",
              "3833 2015-02-24 11:21:16  NEGATIVE  0       \n",
              "3839 2015-02-24 11:05:00  NEGATIVE  2       \n",
              "3844 2015-02-24 10:46:19  NEGATIVE  2       \n",
              "3850 2015-02-24 10:43:21  POSITIVE  1       \n",
              "3851 2015-02-24 10:36:36  NEGATIVE  0       \n",
              "3855 2015-02-24 10:27:09  NEGATIVE  3       \n",
              "3858 2015-02-24 10:22:26  NEGATIVE  4       \n",
              "3860 2015-02-24 10:20:47  POSITIVE  0       "
            ]
          },
          "execution_count": 672,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "not_misc_sw = SW_tweets[SW_tweets['topics'] != -1]\n",
        "not_misc_sw.head(10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7qL5fyloT07e"
      },
      "source": [
        "### Sentiment analysis on each topic for Southwest"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 520
        },
        "id": "eI_xEND8T07g",
        "outputId": "40f14bed-8932-4a12-a204-0a5adf0ac3e6"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-95037e9c-b3fd-4205-84a4-039f40fe0489\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>sentiment</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>topics</th>\n",
              "      <th>sentiment</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">-1</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>69.09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>30.91</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>30.09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>69.91</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>93.30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>6.70</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>89.83</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>10.17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">3</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>41.23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>58.77</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">4</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>85.86</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>14.14</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">5</th>\n",
              "      <th>NEGATIVE</th>\n",
              "      <td>71.43</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>POSITIVE</th>\n",
              "      <td>28.57</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-95037e9c-b3fd-4205-84a4-039f40fe0489')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-95037e9c-b3fd-4205-84a4-039f40fe0489 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-95037e9c-b3fd-4205-84a4-039f40fe0489');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                  sentiment\n",
              "topics sentiment           \n",
              "-1     NEGATIVE   69.09    \n",
              "       POSITIVE   30.91    \n",
              " 0     NEGATIVE   30.09    \n",
              "       POSITIVE   69.91    \n",
              " 1     NEGATIVE   93.30    \n",
              "       POSITIVE   6.70     \n",
              " 2     NEGATIVE   89.83    \n",
              "       POSITIVE   10.17    \n",
              " 3     NEGATIVE   41.23    \n",
              "       POSITIVE   58.77    \n",
              " 4     NEGATIVE   85.86    \n",
              "       POSITIVE   14.14    \n",
              " 5     NEGATIVE   71.43    \n",
              "       POSITIVE   28.57    "
            ]
          },
          "execution_count": 673,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "SW_sentiments = SW_tweets.groupby(['topics', 'sentiment']).agg({'sentiment': 'count'})\n",
        "topics = SW_tweets.groupby(['topics']).agg({'sentiment': 'count'})\n",
        "SW_sentiment_pctg = round(SW_sentiments.div(topics, level='topics') * 100,2)\n",
        "SW_sentiment_pctg\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nf01IDauT07g"
      },
      "source": [
        "Most positive was Topic 0 (twitter actions & thank yous) and the most negative is Topic 1 - flight cancellations"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mjRH2h-SmcKL"
      },
      "source": [
        "## Analyze reason for positive and negative sentiment for each topic and for each airline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6Qqz0abdZAM0"
      },
      "source": [
        "This section aims to get an understanding for the sentiment for each of the topics for each airline by extracting key indicator words (e.g. because) from these tweets and analyzing the output and content"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_iCjWnAzmm9m"
      },
      "outputs": [],
      "source": [
        "becauseUAL_topic0 = UAL_tweets[UAL_tweets.topics == 0]\n",
        "UAL_topic0_p = UAL_topic0[UAL_topic0.sentiment == 'POSITIVE']\n",
        "UAL_topic0_n = UAL_topic0[UAL_topic0.sentiment == 'NEGATIVE']\n",
        "\n",
        "UAL_topic1 = UAL_tweets[UAL_tweets.topics == 1]\n",
        "UAL_topic1_p = UAL_topic1[UAL_topic1.sentiment == 'POSITIVE']\n",
        "UAL_topic1_n = UAL_topic1[UAL_topic1.sentiment == 'NEGATIVE']\n",
        "\n",
        "UAL_topic2 = UAL_tweets[UAL_tweets.topics == 2]\n",
        "UAL_topic2_p = UAL_topic2[UAL_topic2.sentiment == 'POSITIVE']\n",
        "UAL_topic2_n = UAL_topic2[UAL_topic2.sentiment == 'NEGATIVE']\n",
        "\n",
        "UAL_topic3 = UAL_tweets[UAL_tweets.topics == 3]\n",
        "UAL_topic3_p = UAL_topic3[UAL_topic3.sentiment == 'POSITIVE']\n",
        "UAL_topic3_n = UAL_topic3[UAL_topic3.sentiment == 'NEGATIVE']\n",
        "\n",
        "UAL_topic4 = UAL_tweets[UAL_tweets.topics == 4]\n",
        "UAL_topic4_p = UAL_topic4[UAL_topic4.sentiment == 'POSITIVE']\n",
        "UAL_topic4_n = UAL_topic4[UAL_topic4.sentiment == 'NEGATIVE']\n",
        "\n",
        "UAL_topic5 = UAL_tweets[UAL_tweets.topics == 5]\n",
        "UAL_topic5_p = UAL_topic5[UAL_topic5.sentiment == 'POSITIVE']\n",
        "UAL_topic5_n = UAL_topic5[UAL_topic5.sentiment == 'NEGATIVE']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "id": "XJch74eOmm0I",
        "outputId": "34a353fa-f19e-4037-c663-8cd75fedbee5"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-ba7cc3e6-bb71-4ad7-b916-5abcb69f20fc\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>airline</th>\n",
              "      <th>name</th>\n",
              "      <th>text</th>\n",
              "      <th>tweet_created</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>topics</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>10</td>\n",
              "      <td>United</td>\n",
              "      <td>slandail</td>\n",
              "      <td>united gate agent hooked me up with alternate flights. if you have a way to prevent the constant issues that would rock.</td>\n",
              "      <td>2015-02-24 11:19:32</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>38</td>\n",
              "      <td>United</td>\n",
              "      <td>lotusfitness</td>\n",
              "      <td>united newarkliberty airport need to indicate different gate s for terminals a amp c. flyingainteasy</td>\n",
              "      <td>2015-02-24 10:25:12</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53</th>\n",
              "      <td>54</td>\n",
              "      <td>United</td>\n",
              "      <td>kevinfla3</td>\n",
              "      <td>united well sitting on the ground on time but waiting for a gate....again tiredofthis</td>\n",
              "      <td>2015-02-24 09:54:12</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>237</th>\n",
              "      <td>238</td>\n",
              "      <td>United</td>\n",
              "      <td>cristobalwong</td>\n",
              "      <td>united the agent that met us at the gate said any issues w delays on ua would have to be taken up online...</td>\n",
              "      <td>2015-02-23 22:22:53</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>251</th>\n",
              "      <td>252</td>\n",
              "      <td>United</td>\n",
              "      <td>DavidRogerswork</td>\n",
              "      <td>united on arriving two hours late flight into clt. now sitting on the tarmac while they move a plane from the gate. unhappy customer.</td>\n",
              "      <td>2015-02-23 21:46:34</td>\n",
              "      <td>NEGATIVE</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ba7cc3e6-bb71-4ad7-b916-5abcb69f20fc')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ba7cc3e6-bb71-4ad7-b916-5abcb69f20fc button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ba7cc3e6-bb71-4ad7-b916-5abcb69f20fc');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "     tweet_id airline             name  \\\n",
              "9    10        United  slandail          \n",
              "37   38        United  lotusfitness      \n",
              "53   54        United  kevinfla3         \n",
              "237  238       United  cristobalwong     \n",
              "251  252       United  DavidRogerswork   \n",
              "\n",
              "                                                                                                                                      text  \\\n",
              "9    united gate agent hooked me up with alternate flights. if you have a way to prevent the constant issues that would rock.                \n",
              "37   united newarkliberty airport need to indicate different gate s for terminals a amp c. flyingainteasy                                    \n",
              "53   united well sitting on the ground on time but waiting for a gate....again tiredofthis                                                   \n",
              "237  united the agent that met us at the gate said any issues w delays on ua would have to be taken up online...                             \n",
              "251  united on arriving two hours late flight into clt. now sitting on the tarmac while they move a plane from the gate. unhappy customer.   \n",
              "\n",
              "          tweet_created sentiment  topics  \n",
              "9   2015-02-24 11:19:32  NEGATIVE  5       \n",
              "37  2015-02-24 10:25:12  NEGATIVE  5       \n",
              "53  2015-02-24 09:54:12  NEGATIVE  5       \n",
              "237 2015-02-23 22:22:53  NEGATIVE  5       \n",
              "251 2015-02-23 21:46:34  NEGATIVE  5       "
            ]
          },
          "execution_count": 675,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "UAL_topic5_n.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PICKhrT8qutf"
      },
      "source": [
        "#### Running for United Airlines all poitive topics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NgqlP4drOkDh",
        "outputId": "cffe9af2-3c6d-4408-d14a-00c2763e6129"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'emailed', 'virginatlantic', 'found', 'left', 'lhr', 'mean', 'time']\n"
          ]
        }
      ],
      "source": [
        "data_list = UAL_topic0_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''   \n",
        "    for w in i:        \n",
        "        a = w+\" \"        \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uMwEZG9RoP0c"
      },
      "outputs": [],
      "source": [
        "data_list = UAL_topic1_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "from nltk.stem import PorterStemmer \n",
        "ps = PorterStemmer()\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    \n",
        "    for w in i:\n",
        "        \n",
        "        a = w+\" \"\n",
        "        \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JYR9cC6T0QOx"
      },
      "outputs": [],
      "source": [
        "data_list = UAL_topic2_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''   \n",
        "    for w in i:        \n",
        "        a = w+\" \"        \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FCLBvYar0P_p"
      },
      "outputs": [],
      "source": [
        "data_list = UAL_topic3_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''   \n",
        "    for w in i:        \n",
        "        a = w+\" \"        \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tvp5hmy10Pvw"
      },
      "outputs": [],
      "source": [
        "data_list = UAL_topic4_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''   \n",
        "    for w in i:        \n",
        "        a = w+\" \"        \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lxP1DxAFoWcU"
      },
      "outputs": [],
      "source": [
        "data_list = UAL_topic5_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "from nltk.stem import PorterStemmer \n",
        "ps = PorterStemmer()\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    \n",
        "    for w in i:\n",
        "        \n",
        "        a = w+\" \"\n",
        "        \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2DfyvJnJq537"
      },
      "source": [
        "#### Running for United Airlines all negative topics\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jvUX-vrLbono"
      },
      "source": [
        "#### Topic 0 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dS4dki5Nokhh",
        "outputId": "72705c31-98bc-43c5-def1-8ab2cae5d6e0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'flight', 'staff', 'police', 'overhead', 'bins']\n",
            "['because', 'missed', 'connection']\n",
            "['because', 'plane', 'overweight', 'great', 'knew', 'soon', 'landed', 'instead']\n",
            "['because', 'wait', 'bag', 'baggage', 'claim']\n",
            "['because']\n",
            "['because', 'delayed', 'flight', 'resulted', 'bags', 'rechecked', 'overnight']\n",
            "['because', 'carry', 'bags', 'defend', 'free']\n",
            "['because', 'happens', 'every', 'week', 'sister', 'travels', 'airline']\n",
            "['because']\n"
          ]
        }
      ],
      "source": [
        "data_list = UAL_topic0_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "from nltk.stem import PorterStemmer \n",
        "ps = PorterStemmer()\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    \n",
        "    for w in i:\n",
        "        \n",
        "        a = w+\" \"\n",
        "        \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WWr_STYSb2ec"
      },
      "source": [
        "#### Topic 1 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j3VM33ydopkm",
        "outputId": "8310fd18-26d7-40b6-ba12-500584322ac1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'head', 'pounding']\n",
            "['because', 'flight', 'take']\n"
          ]
        }
      ],
      "source": [
        "data_list = UAL_topic1_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "from nltk.stem import PorterStemmer \n",
        "ps = PorterStemmer()\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    \n",
        "    for w in i:\n",
        "        \n",
        "        a = w+\" \"\n",
        "        \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xII5H298b_Vv"
      },
      "source": [
        "#### Topic 2 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B4M2gINDou_9",
        "outputId": "1068bfdf-0621-4b7f-d417-bbb633432162"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'glitch', 'notfair']\n",
            "['because', 'last', 'minute', 'cancelled', 'flightled', 'connecting', 'flight', 'booked']\n",
            "['because', 'empty', 'seats', 'confirmed', 'important', 'frustrated']\n",
            "['because', 'crew', 'timed', 'right', 'hr', 'flight']\n",
            "['because', 'pilot', 'miss', 'meeting', 'atlanta', 'because', 'someone', 'slept']\n",
            "['because', 'someone', 'slept']\n"
          ]
        }
      ],
      "source": [
        "data_list = UAL_topic2_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "from nltk.stem import PorterStemmer \n",
        "ps = PorterStemmer()\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z77QQhXUcCph"
      },
      "source": [
        "#### Topic 3 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4eg5cjNdp5kQ",
        "outputId": "113dac18-1691-40a4-f2d3-7db79bb249fa"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because']\n"
          ]
        }
      ],
      "source": [
        "data_list = UAL_topic3_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''    \n",
        "    for w in i:        \n",
        "        a = w+\" \"     \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kBFBNgyEcF0N"
      },
      "source": [
        "#### Topic 4 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EmALAziJp5ao",
        "outputId": "29b4c0e8-7669-43b3-92d8-942479a06b80"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'missed', 'ride', 'because', 'hour', 'delay', 'getmartyhome']\n",
            "['because', 'hour', 'delay', 'getmartyhome']\n",
            "['because', 'print', 'piece', 'paper']\n",
            "['because', 'copilot', 'hours', 'flying', 'raleigh', 'disappointing']\n",
            "['because', 'maintenance', 'fixed', 'board', 'because', 'flight', 'crew', 'stay']\n",
            "['because', 'flight', 'crew', 'stay', 'boarding', 'area', 'fail']\n",
            "['because', 'charge', 'delay', 'united', 'still', 'waiting']\n",
            "['because', 'hour', 'delay', 'first', 'flight', 'justgetmehome']\n",
            "['because', 'plane', 'food', 'fixed', 'future', 'dumb', 'delay']\n",
            "['because', 'inbound', 'int', 'flights', 'crazy']\n",
            "['because', 'plane', 'frozen', 'fail', 'make', 'connection', 'ua', 'tightconnection']\n",
            "['because', 'pilot', 'never', 'showed']\n"
          ]
        }
      ],
      "source": [
        "data_list = UAL_topic4_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''    \n",
        "    for w in i:        \n",
        "        a = w+\" \"     \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZiXIvP5jcIxt"
      },
      "source": [
        "#### Topic 5 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bjiAIVo8p5Ps",
        "outputId": "f1530c90-b1a6-435a-fb31-f858baf5f4a0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'jetway', 'broken']\n",
            "['because', 'someone', 'corporate', 'dispatch', 'forgot', 'call', 'crew', 'unfriendlyskies']\n"
          ]
        }
      ],
      "source": [
        "data_list = UAL_topic5_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''    \n",
        "    for w in i:        \n",
        "        a = w+\" \"     \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n2dmc2cbrKRq"
      },
      "source": [
        "### American Airline "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dYFHiU48rJnN"
      },
      "outputs": [],
      "source": [
        "AA_topic0 = AA_tweets[AA_tweets.topics == 0]\n",
        "AA_topic0_p = AA_topic0[AA_topic0.sentiment == 'POSITIVE']\n",
        "AA_topic0_n = AA_topic0[AA_topic0.sentiment == 'NEGATIVE']\n",
        "\n",
        "AA_topic1 = AA_tweets[AA_tweets.topics == 1]\n",
        "AA_topic1_p = AA_topic1[AA_topic1.sentiment == 'POSITIVE']\n",
        "AA_topic1_n = AA_topic1[AA_topic1.sentiment == 'NEGATIVE']\n",
        "\n",
        "AA_topic2 = AA_tweets[AA_tweets.topics == 2]\n",
        "AA_topic2_p = AA_topic2[AA_topic2.sentiment == 'POSITIVE']\n",
        "AA_topic2_n = AA_topic2[AA_topic2.sentiment == 'NEGATIVE']\n",
        "\n",
        "AA_topic3 = AA_tweets[AA_tweets.topics == 3]\n",
        "AA_topic3_p = AA_topic3[AA_topic3.sentiment == 'POSITIVE']\n",
        "AA_topic3_n = AA_topic3[AA_topic3.sentiment == 'NEGATIVE']\n",
        "\n",
        "AA_topic4 = AA_tweets[AA_tweets.topics == 4]\n",
        "AA_topic4_p = AA_topic4[AA_topic4.sentiment == 'POSITIVE']\n",
        "AA_topic4_n = AA_topic4[AA_topic4.sentiment == 'NEGATIVE']\n",
        "\n",
        "AA_topic5 = AA_tweets[AA_tweets.topics == 5]\n",
        "AA_topic5_p = AA_topic5[AA_topic5.sentiment == 'POSITIVE']\n",
        "AA_topic5_n = AA_topic5[AA_topic5.sentiment == 'NEGATIVE']"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P_jqAWO7r_Ck"
      },
      "source": [
        "#### Positive American Airlines"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "llhTFXMzrJdZ"
      },
      "outputs": [],
      "source": [
        "data_list = AA_topic0_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a4U2GjlArJZJ",
        "outputId": "b9deea0b-fa2c-4dd5-a72f-3ff3c2666f70"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'one', 'thing', 'tried', 'avoid', 'thank', 'sending', 'baggage']\n"
          ]
        }
      ],
      "source": [
        "data_list = AA_topic1_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "953ybxi_rJSo"
      },
      "outputs": [],
      "source": [
        "data_list = AA_topic2_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7f9q3DYsrJBp"
      },
      "outputs": [],
      "source": [
        "data_list = AA_topic3_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9JkF4m3JsKOx"
      },
      "outputs": [],
      "source": [
        "data_list = AA_topic4_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "42_zgj-LsKMQ"
      },
      "outputs": [],
      "source": [
        "data_list = AA_topic5_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        " \n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N_NdsbZSsX5J"
      },
      "source": [
        "#### Negative American Airline topics"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pacewKv7cl-p"
      },
      "source": [
        "#### Topic 0 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EXvn0sz6sKJo",
        "outputId": "db113990-bd92-4ca3-cb5d-c803d05535ec"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'worked']\n",
            "['because', 'automated', 'system', 'keeps', 'hanging', 'two', 'days', 'help']\n",
            "['because', 'issues', 'weather', 'today']\n",
            "['because', 'busy', 'literally', 'disconnect', 'bother']\n"
          ]
        }
      ],
      "source": [
        "data_list = AA_topic0_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ys2Dfz2ZcoFo"
      },
      "source": [
        "#### Topic 1 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S-AKfJg3sKHP",
        "outputId": "f8fcdb27-e51a-4b1d-f36d-88e1252b5520"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'says', 'located', 'yet']\n",
            "['because', 'get', 'bags', 'us', 'because', 'said', 'would', 'go']\n",
            "['because', 'said', 'would', 'go', 'cincinnati', 'even', 'though', 'exactly']\n"
          ]
        }
      ],
      "source": [
        "data_list = AA_topic1_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e9uQ1AMXcqbZ"
      },
      "source": [
        "#### Topic 2 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "427T-fYvsKEg"
      },
      "outputs": [],
      "source": [
        "data_list = AA_topic2_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-Spwa1n9cs2E"
      },
      "source": [
        "#### Topic 3 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "emMxvYEXsKBr",
        "outputId": "0ad588ca-28dd-4b14-c057-d347f0d2a861"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'high', 'volume', 'customer', 'service', 'tried', 'resolve', 'via']\n"
          ]
        }
      ],
      "source": [
        "data_list = AA_topic3_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VK88zFyQcwci"
      },
      "source": [
        "#### Topic 4 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zhmWymIssJ-x",
        "outputId": "046a03a8-a3c5-4430-8438-b13f512d1207"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'airline', 'sucks']\n",
            "['because']\n"
          ]
        }
      ],
      "source": [
        "data_list = AA_topic4_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xq0eD-8LczV3"
      },
      "source": [
        "#### Topic 5 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iDhh6LIEskq2",
        "outputId": "649311ed-d8b2-4814-9677-58aee0f7de0d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'last', 'name', 'right', 'thing', 'reinstate', 'tickets']\n",
            "['because', 'weather', 'dallas', 'without', 'charged']\n",
            "['because', 'weather', 'lied', 'us', 'said', 'airlines', 'cancelled', 'flightled']\n"
          ]
        }
      ],
      "source": [
        "data_list = AA_topic5_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6Oz_-OwVskOv"
      },
      "source": [
        "#### Southwest Airlines"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lqX9tLfbswhN"
      },
      "outputs": [],
      "source": [
        "SW_topic0 = SW_tweets[SW_tweets.topics == 0]\n",
        "SW_topic0_p = SW_topic0[SW_topic0.sentiment == 'POSITIVE']\n",
        "SW_topic0_n = SW_topic0[SW_topic0.sentiment == 'NEGATIVE']\n",
        "\n",
        "SW_topic1 = SW_tweets[SW_tweets.topics == 1]\n",
        "SW_topic1_p = SW_topic1[SW_topic1.sentiment == 'POSITIVE']\n",
        "SW_topic1_n = SW_topic1[SW_topic1.sentiment == 'NEGATIVE']\n",
        "\n",
        "SW_topic2 = SW_tweets[SW_tweets.topics == 2]\n",
        "SW_topic2_p = SW_topic2[SW_topic2.sentiment == 'POSITIVE']\n",
        "SW_topic2_n = SW_topic2[SW_topic2.sentiment == 'NEGATIVE']\n",
        "\n",
        "SW_topic3 = SW_tweets[SW_tweets.topics == 3]\n",
        "SW_topic3_p = SW_topic3[SW_topic3.sentiment == 'POSITIVE']\n",
        "SW_topic3_n = SW_topic3[SW_topic3.sentiment == 'NEGATIVE']\n",
        "\n",
        "SW_topic4 = SW_tweets[SW_tweets.topics == 4]\n",
        "SW_topic4_p = SW_topic4[SW_topic4.sentiment == 'POSITIVE']\n",
        "SW_topic4_n = SW_topic4[SW_topic4.sentiment == 'NEGATIVE']\n",
        "\n",
        "SW_topic5 = SW_tweets[SW_tweets.topics == 5]\n",
        "SW_topic5_p = SW_topic5[SW_topic5.sentiment == 'POSITIVE']\n",
        "SW_topic5_n = SW_topic5[SW_topic5.sentiment == 'NEGATIVE']"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pFRwuVAqt2oH"
      },
      "source": [
        "Positive Southwest topics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pskmi1tbsux_"
      },
      "outputs": [],
      "source": [
        "data_list = SW_topic0_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "W2WSM2gGsuo4"
      },
      "outputs": [],
      "source": [
        "data_list = SW_topic1_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XDpAv9pVsufw"
      },
      "outputs": [],
      "source": [
        "data_list = SW_topic2_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c04Wx7oMsuV3",
        "outputId": "7701a237-da1b-456d-d0d1-6215c898d597"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'imaginedragons', 'chance', 'could', 'get', 'tickets']\n"
          ]
        }
      ],
      "source": [
        "data_list = SW_topic3_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_IUXAHoDsuNR"
      },
      "outputs": [],
      "source": [
        "data_list = SW_topic4_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XpuiLKpjsuEK"
      },
      "outputs": [],
      "source": [
        "data_list = SW_topic5_p['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7xxRCdtouGiQ"
      },
      "source": [
        "#### Negative Southwest topics"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NQQSmnawc92O"
      },
      "source": [
        "#### Topic 0 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BSuKEi7Rst6R",
        "outputId": "585c0898-3d79-464d-fe5c-b91b70edee06"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'follow', 'notanymore', 'wrong', 'badpolicy']\n",
            "['because', 'follow']\n",
            "['because', 'need', 'follow', 'back']\n"
          ]
        }
      ],
      "source": [
        "data_list = SW_topic0_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BY0hmek1c_nQ"
      },
      "source": [
        "#### Topic 1 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VD4OHQP4stvm",
        "outputId": "4e33d8d4-1fef-41bb-b084-2c2a0b77bf0b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'flight', 'cancelled', 'flightled', 'apparent', 'reason', 'told', 'sorry']\n",
            "['because', 'yall', 'cancelled', 'flightled', 'return', 'flight', 'answer', 'phone']\n",
            "['because', 'cavalli', 'calves', 'interfering', 'onboard', 'equipment']\n",
            "['because', 'responsible', 'adult', 'got', 'number', 'call', 'ty']\n"
          ]
        }
      ],
      "source": [
        "data_list = SW_topic1_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qSeMlI1bdB54"
      },
      "source": [
        "#### Topic 2 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i-YUQvcXstlb",
        "outputId": "307e66b5-fcac-4816-fc0b-74059637a01e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'phone', 'dying', 'customerservice']\n"
          ]
        }
      ],
      "source": [
        "data_list = SW_topic2_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RHyuEsIzdDxK"
      },
      "source": [
        "#### Topic 3 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "86mHFy-FstZv"
      },
      "outputs": [],
      "source": [
        "data_list = SW_topic3_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w5LXkQy_dG0s"
      },
      "source": [
        "#### Topic 4 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uqtbhXFAstQF",
        "outputId": "b5ec42f7-2b2b-45b3-d8ad-a22f08d5d834"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['because', 'badpolicy', 'allows', 'cheatcustomers']\n",
            "['because', 'volume', 'high']\n",
            "['because', 'bags', 'loaded', 'correctly', 'gives']\n"
          ]
        }
      ],
      "source": [
        "data_list = SW_topic4_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4wgRzowkdJt6"
      },
      "source": [
        "#### Topic 5 -- Negative Reasons"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4k3AseZFss9E"
      },
      "outputs": [],
      "source": [
        "data_list = SW_topic5_n['text'].tolist()        \n",
        "\n",
        "add_stopwords = ['www', 'com','http','https', 'jpg','subscribe']\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english'))- set(['because']))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stopwords] for doc in texts]\n",
        "\n",
        "# Remove Stop Words\n",
        "data_nostops = remove_stopwords(data_list)\n",
        "\n",
        "# choose some words to be stemmed\n",
        "news_clean = []\n",
        "\n",
        "for i in data_nostops:\n",
        "    string = ''\n",
        "    for w in i:\n",
        "        a = w+\" \"    \n",
        "        string += a\n",
        "    news_clean.append(string)\n",
        "\n",
        "for i in data_nostops:\n",
        "    for j in range(len(i)):\n",
        "    \n",
        "        if i[j]=='because':\n",
        "            print(i[j:j+8])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VFd8Ot5mOkDg"
      },
      "source": [
        "### Keyword Extraction & Biagrams -- For Positive & Negative UAL Sentiments"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wRqbVOP9fc0G"
      },
      "source": [
        "Extract most commonly negatively or positively associated words and bi-grams for UAL"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M4i2ijqp8yvZ"
      },
      "source": [
        "#### UAL Negative associated words/Phrase\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Gn8qFsUTOkDg"
      },
      "outputs": [],
      "source": [
        "top_N = 100\n",
        "\n",
        "negative_ual = UAL_tweets[tweets['sentiment'] == \"NEGATIVE\"]\n",
        "\n",
        "tweet_text = negative_ual['text'].str.lower().str.replace(r'\\|', ' ', regex=True).str.cat(sep=' ')\n",
        "\n",
        "\n",
        "# Use TweetTokenizer\n",
        "tweet_tokenizer = nltk.tokenize.TweetTokenizer()\n",
        "words = tweet_tokenizer.tokenize(tweet_text)\n",
        "\n",
        "#stopwords = stopwords.words('english')\n",
        "add_stopwords = [\"united\",\"americanair\",\"southwestair\",\"flight\",\"flights\",\"fly\"]\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english')))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Remove single-character tokens (mostly punctuation)\n",
        "words = [word for word in words if len(word) > 1]\n",
        "\n",
        "# Remove numbers\n",
        "words = [word for word in words if not word.isnumeric()]\n",
        "\n",
        "# Remove punctuation\n",
        "# words = [word for word in words if word.isalpha()]\n",
        "\n",
        "# Lowercase all words (default_stopwords are lowercase too)\n",
        "words = [word.lower() for word in words]\n",
        "\n",
        "# Remove stopwords\n",
        "words = [word for word in words if word not in stopwords]\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "duFdRV73OkDh"
      },
      "outputs": [],
      "source": [
        "word_dist = nltk.FreqDist(words)\n",
        "\n",
        "df_word_dist = pd.DataFrame(word_dist.most_common(top_N),\n",
        "                    columns=['Word', 'Frequency']).set_index('Word')\n",
        "\n",
        "\n",
        "df_word_dist.to_csv('/content/gdrive/MyDrive/Machine Learning/Final/ual_neg_word.csv')\n",
        "df_word_dist.head(10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dcwVWbCef8zJ"
      },
      "source": [
        "##### Bi-grams"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "bLKmWMQNf6n5",
        "outputId": "9f3d177b-8450-4963-fa56-db66af2c32f8"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-e0da12a7-36a7-4c0f-a014-e5470ca308fd\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Word</th>\n",
              "      <th>Frequency</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>(http, t.co)</td>\n",
              "      <td>179</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>(customer, service)</td>\n",
              "      <td>127</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>(cancelled, flightled)</td>\n",
              "      <td>84</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>(late, flightr)</td>\n",
              "      <td>41</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>(booking, problems)</td>\n",
              "      <td>37</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>(cancelled, flighted)</td>\n",
              "      <td>35</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>(gate, agent)</td>\n",
              "      <td>21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>(worst, airline)</td>\n",
              "      <td>19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>(hour, delay)</td>\n",
              "      <td>18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>(trying, get)</td>\n",
              "      <td>17</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e0da12a7-36a7-4c0f-a014-e5470ca308fd')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e0da12a7-36a7-4c0f-a014-e5470ca308fd button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e0da12a7-36a7-4c0f-a014-e5470ca308fd');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                     Word  Frequency\n",
              "0  (http, t.co)            179      \n",
              "1  (customer, service)     127      \n",
              "2  (cancelled, flightled)  84       \n",
              "3  (late, flightr)         41       \n",
              "4  (booking, problems)     37       \n",
              "5  (cancelled, flighted)   35       \n",
              "6  (gate, agent)           21       \n",
              "7  (worst, airline)        19       \n",
              "8  (hour, delay)           18       \n",
              "9  (trying, get)           17       "
            ]
          },
          "execution_count": 65,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "bgs = nltk.bigrams(words)\n",
        "\n",
        "#compute frequency distribution for all the bigrams in the text\n",
        "fdist_2 = nltk.FreqDist(bgs)\n",
        "\n",
        "fdist_2_df = pd.DataFrame(fdist_2.most_common(),\n",
        "                    columns=['Word', 'Frequency'])\n",
        "\n",
        "fdist_2_df.head(n=30)\n",
        "\n",
        "fdist_2_df.to_csv('/content/gdrive/MyDrive/Machine Learning/Final/ual_neg_bigrams.csv')\n",
        "fdist_2_df.head(10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jmHEKv0FgDux"
      },
      "source": [
        "##### Trigrams"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "bh9QBBZiOkDh",
        "outputId": "49910239-7b7e-41ea-bb6c-d752f21b2610"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-113cd0c6-6ae1-48ca-9a53-3733ceabd8c7\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Word</th>\n",
              "      <th>Frequency</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>(reflight, booking, problems)</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>(late, flightr, still)</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>(hours, late, flightr)</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>(..., http, t.co)</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>(lt, lt, lt)</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>(worst, service, ever)</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>(worst, airline, ever)</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>(worst, customer, service)</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>(terrible, customer, service)</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>(hrs, late, flightr)</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-113cd0c6-6ae1-48ca-9a53-3733ceabd8c7')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-113cd0c6-6ae1-48ca-9a53-3733ceabd8c7 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-113cd0c6-6ae1-48ca-9a53-3733ceabd8c7');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                            Word  Frequency\n",
              "0  (reflight, booking, problems)  12       \n",
              "1  (late, flightr, still)         11       \n",
              "2  (hours, late, flightr)         11       \n",
              "3  (..., http, t.co)              8        \n",
              "4  (lt, lt, lt)                   8        \n",
              "5  (worst, service, ever)         7        \n",
              "6  (worst, airline, ever)         7        \n",
              "7  (worst, customer, service)     6        \n",
              "8  (terrible, customer, service)  6        \n",
              "9  (hrs, late, flightr)           5        "
            ]
          },
          "execution_count": 66,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "bgs = nltk.trigrams(words)\n",
        "\n",
        "#compute frequency distribution for all the bigrams in the text\n",
        "fdist_3 = nltk.FreqDist(bgs)\n",
        "\n",
        "fdist_3_df = pd.DataFrame(fdist_3.most_common(),\n",
        "                    columns=['Word', 'Frequency'])\n",
        "\n",
        "fdist_3_df.to_csv('/content/gdrive/MyDrive/Machine Learning/Final/ual_neg_trigrams.csv')\n",
        "fdist_3_df.head(10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B_1R9K5C9RE3"
      },
      "source": [
        "#### UAL Positive Asssociated words/phrase\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1NXylPPofWwr",
        "outputId": "17e948d6-f02f-4f65-ec20-bcf6c99e44ce"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting yake\n",
            "  Downloading yake-0.4.8-py2.py3-none-any.whl (60 kB)\n",
            "\u001b[K     |████████████████████████████████| 60 kB 4.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.8/dist-packages (from yake) (1.21.6)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.8/dist-packages (from yake) (2.8.8)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.8/dist-packages (from yake) (0.8.10)\n",
            "Collecting jellyfish\n",
            "  Downloading jellyfish-0.9.0.tar.gz (132 kB)\n",
            "\u001b[K     |████████████████████████████████| 132 kB 14.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: click>=6.0 in /usr/local/lib/python3.8/dist-packages (from yake) (7.1.2)\n",
            "Collecting segtok\n",
            "  Downloading segtok-1.5.11-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.8/dist-packages (from segtok->yake) (2022.6.2)\n",
            "Building wheels for collected packages: jellyfish\n",
            "  Building wheel for jellyfish (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for jellyfish: filename=jellyfish-0.9.0-cp38-cp38-linux_x86_64.whl size=70609 sha256=90e7a72c60a4d153844e53a60a6dc8eda451c39850b8b38eaae3801dbc50c15d\n",
            "  Stored in directory: /root/.cache/pip/wheels/f1/c7/3c/4c83132de76359e3a429fd09c08995945ca96c5290a41651d3\n",
            "Successfully built jellyfish\n",
            "Installing collected packages: segtok, jellyfish, yake\n",
            "Successfully installed jellyfish-0.9.0 segtok-1.5.11 yake-0.4.8\n"
          ]
        }
      ],
      "source": [
        "pip install yake"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x9ieL5zkOkDh",
        "outputId": "78a9b689-73b2-4eb8-9615-146ae10caae8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "('united thanks for taking', -5.795284019359414)\n",
            "('united', 0.02616467041449924)\n",
            "('happy customer', 0.03818705505179179)\n",
            "('taking care', 0.044207092268886417)\n",
            "('care of that mr happy', 0.044207092268886417)\n",
            "('customer', 0.178767226439438)\n",
            "('taking', 0.20575606977834637)\n",
            "('care', 0.20575606977834637)\n",
            "('happy', 0.20575606977834637)\n",
            "('eri', 0.24999444691704367)\n",
            "('united your eri ord express', 0.29491373606901705)\n",
            "('business', 0.3248150296169627)\n",
            "('united your eri ord', 0.3335966641256541)\n",
            "('eri ord express', 0.3717814253011423)\n",
            "('eri ord express connections', 0.3796638089585924)\n",
            "('united thanks for taking care', 0.3883747822883905)\n",
            "('Length', 0.3892283460464486)\n",
            "('eri ord', 0.39591268852066963)\n",
            "('connections are hugely popular', 0.4007239767942396)\n",
            "('express connections are hugely popular', 0.41117278696290366)\n",
            "('hugely popular', 0.4231328968166318)\n",
            "('eri ewr hop', 0.4375034320899209)\n",
            "('eri ewr', 0.4454230826430665)\n",
            "('ord express connections', 0.45665191958096374)\n",
            "('express connections are hugely', 0.45665191958096374)\n",
            "('ord express connections are hugely', 0.4729199783606306)\n",
            "('filled', 0.47360912997492655)\n",
            "('ord express', 0.4744519572880905)\n",
            "('express connections', 0.4744519572880905)\n",
            "('connections are hugely', 0.4744519572880905)\n",
            "('united your eri', 0.4920395939177339)\n",
            "('common', 0.5241438872442727)\n",
            "('popular', 0.5241438872442727)\n",
            "('ewr hop', 0.5481066763006778)\n",
            "('luck', 0.5672582334352227)\n",
            "('ord', 0.5672582334352227)\n",
            "('express', 0.5672582334352227)\n",
            "('connections', 0.5672582334352227)\n",
            "('hugely', 0.5672582334352227)\n",
            "('united new apple', 0.5729581894729303)\n",
            "('hop', 0.5744428559883982)\n",
            "('united even though technically', 0.5754753057937485)\n",
            "('ticket', 0.6064366907806417)\n",
            "('ewr', 0.6163357733759776)\n",
            "('hare', 0.6456449818027716)\n",
            "('text', 0.6498966471360186)\n",
            "('dtype', 0.6498966471360186)\n",
            "('object', 0.6498966471360186)\n",
            "('comfortable', 0.6587540573828606)\n",
            "('enjoyable', 0.6693465929468384)\n"
          ]
        }
      ],
      "source": [
        "import yake\n",
        "UAL_positive = UAL_tweets[UAL_tweets['sentiment'] == \"POSITIVE\"]\n",
        "kw_extractor = yake.KeywordExtractor()\n",
        "text = str(UAL_positive['text'])\n",
        "language = \"en\"\n",
        "max_ngram_size = 5\n",
        "deduplication_threshold = 0.9\n",
        "numOfKeywords = 50\n",
        "custom_kw_extractor = yake.KeywordExtractor(lan=language, n=max_ngram_size, dedupLim=deduplication_threshold, top=numOfKeywords, features=None)\n",
        "keywords = custom_kw_extractor.extract_keywords(text)\n",
        "for kw in keywords:\n",
        "     print(kw)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HgiyasV_9RE4"
      },
      "outputs": [],
      "source": [
        "top_N = 100\n",
        "\n",
        "\n",
        "tweet_text = UAL_positive['text'].str.lower().str.replace(r'\\|', ' ', regex=True).str.cat(sep=' ')\n",
        "\n",
        "\n",
        "# Use TweetTokenizer\n",
        "tweet_tokenizer = nltk.tokenize.TweetTokenizer()\n",
        "words = tweet_tokenizer.tokenize(tweet_text)\n",
        "\n",
        "#stopwords = stopwords.words('english')\n",
        "add_stopwords = [\"united\",\"americanair\",\"southwestair\",\"flight\",\"flights\",\"fly\"]\n",
        "stopwords = list(set(nltk.corpus.stopwords.words('english')))\n",
        "for i in add_stopwords:     \n",
        "    stopwords.append(i)\n",
        "\n",
        "# Remove single-character tokens (mostly punctuation)\n",
        "words = [word for word in words if len(word) > 1]\n",
        "\n",
        "# Remove numbers\n",
        "words = [word for word in words if not word.isnumeric()]\n",
        "\n",
        "# Remove punctuation\n",
        "# words = [word for word in words if word.isalpha()]\n",
        "\n",
        "# Lowercase all words (default_stopwords are lowercase too)\n",
        "words = [word.lower() for word in words]\n",
        "\n",
        "# Remove stopwords\n",
        "words = [word for word in words if word not in stopwords]\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 394
        },
        "id": "bngm-2Um9RE4",
        "outputId": "231a5f58-5271-46c2-b6aa-2e60da9039b0"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-b10e5957-3843-4cb8-9425-c19cf0ede9d6\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Frequency</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Word</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>thanks</th>\n",
              "      <td>185</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>thank</th>\n",
              "      <td>128</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>get</th>\n",
              "      <td>61</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>t.co</th>\n",
              "      <td>60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>http</th>\n",
              "      <td>57</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>service</th>\n",
              "      <td>56</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>49</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>great</th>\n",
              "      <td>47</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>customer</th>\n",
              "      <td>45</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>good</th>\n",
              "      <td>44</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b10e5957-3843-4cb8-9425-c19cf0ede9d6')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b10e5957-3843-4cb8-9425-c19cf0ede9d6 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b10e5957-3843-4cb8-9425-c19cf0ede9d6');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "          Frequency\n",
              "Word               \n",
              "thanks    185      \n",
              "thank     128      \n",
              "get       61       \n",
              "t.co      60       \n",
              "http      57       \n",
              "service   56       \n",
              "...       49       \n",
              "great     47       \n",
              "customer  45       \n",
              "good      44       "
            ]
          },
          "execution_count": 68,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "word_dist = nltk.FreqDist(words)\n",
        "\n",
        "df_word_dist = pd.DataFrame(word_dist.most_common(top_N),\n",
        "                    columns=['Word', 'Frequency']).set_index('Word')\n",
        "\n",
        "\n",
        "df_word_dist.to_csv('/content/gdrive/MyDrive/Machine Learning/Final/ual_pos_words.csv')\n",
        "df_word_dist.head(10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BXNQRDMA9RE4"
      },
      "source": [
        "##### Bi-grams"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "OV5BY_IJ9RE4",
        "outputId": "e8b1e5eb-0847-4998-8500-8c2347ebefbe"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-6f77ff1b-c773-4b82-9fe0-7bd1edec4cb6\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Word</th>\n",
              "      <th>Frequency</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>(http, t.co)</td>\n",
              "      <td>57</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>(customer, service)</td>\n",
              "      <td>29</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>(first, class)</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>(thank, much)</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>(let, know)</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>(st, class)</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>(make, sure)</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>(thanks, help)</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>(get, home)</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>(cancelled, flightled)</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6f77ff1b-c773-4b82-9fe0-7bd1edec4cb6')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6f77ff1b-c773-4b82-9fe0-7bd1edec4cb6 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6f77ff1b-c773-4b82-9fe0-7bd1edec4cb6');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                     Word  Frequency\n",
              "0  (http, t.co)            57       \n",
              "1  (customer, service)     29       \n",
              "2  (first, class)          12       \n",
              "3  (thank, much)           10       \n",
              "4  (let, know)             9        \n",
              "5  (st, class)             8        \n",
              "6  (make, sure)            8        \n",
              "7  (thanks, help)          7        \n",
              "8  (get, home)             7        \n",
              "9  (cancelled, flightled)  7        "
            ]
          },
          "execution_count": 69,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "bgs = nltk.bigrams(words)\n",
        "\n",
        "#compute frequency distribution for all the bigrams in the text\n",
        "fdist_2 = nltk.FreqDist(bgs)\n",
        "\n",
        "fdist_2_df = pd.DataFrame(fdist_2.most_common(),\n",
        "                    columns=['Word', 'Frequency'])\n",
        "\n",
        "fdist_2_df.to_csv('/content/gdrive/MyDrive/Machine Learning/Final/ual_pos_bigrams.csv')\n",
        "fdist_2_df.head(10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Kon8_vAf9RE4"
      },
      "source": [
        "##### Trigrams"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "yW8xe-oV9RE4",
        "outputId": "da7c5368-64b5-4277-aa2d-a024231e8170"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-1c03a9ce-9b97-4190-a7a6-e0eade236c9a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Word</th>\n",
              "      <th>Frequency</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>(great, customer, service)</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>(keep, good, work)</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>(let, us, know)</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>(reflight, booking, problems)</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>(dm, sent, thanks)</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>(thank, much, help)</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>(global, first, class)</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>(contact, customer, care)</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>(awesome, new, plane)</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>(give, shot, thank)</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1c03a9ce-9b97-4190-a7a6-e0eade236c9a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1c03a9ce-9b97-4190-a7a6-e0eade236c9a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1c03a9ce-9b97-4190-a7a6-e0eade236c9a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                            Word  Frequency\n",
              "0  (great, customer, service)     4        \n",
              "1  (keep, good, work)             3        \n",
              "2  (let, us, know)                3        \n",
              "3  (reflight, booking, problems)  3        \n",
              "4  (dm, sent, thanks)             3        \n",
              "5  (thank, much, help)            3        \n",
              "6  (global, first, class)         3        \n",
              "7  (contact, customer, care)      2        \n",
              "8  (awesome, new, plane)          2        \n",
              "9  (give, shot, thank)            2        "
            ]
          },
          "execution_count": 70,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "bgs = nltk.trigrams(words)\n",
        "\n",
        "#compute frequency distribution for all the bigrams in the text\n",
        "fdist_3 = nltk.FreqDist(bgs)\n",
        "\n",
        "fdist_3_df = pd.DataFrame(fdist_3.most_common(),\n",
        "                    columns=['Word', 'Frequency'])\n",
        "\n",
        "\n",
        "fdist_3_df.to_csv('/content/gdrive/MyDrive/Machine Learning/Final/ual_pos_trigrams.csv')\n",
        "fdist_3_df.head(10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1gb2kl_nOkDf"
      },
      "source": [
        "### UAL Sentiment Analysis Over Time and Visualization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iOO-UgWiOkDf"
      },
      "outputs": [],
      "source": [
        "for_plot = UAL_tweets.groupby(['tweet_created','sentiment'])['sentiment'].count()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aMdnj0ONOkDf"
      },
      "outputs": [],
      "source": [
        "for_plot = pd.DataFrame(for_plot)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8kECxUS4OkDf"
      },
      "outputs": [],
      "source": [
        "for_plot.columns = ['sentiment_counts']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hIlywE6pOkDf"
      },
      "outputs": [],
      "source": [
        "for_plot = for_plot.reset_index()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3_mLMX7NOkDf"
      },
      "outputs": [],
      "source": [
        "for_plot['tweet_created'] = pd.to_datetime(for_plot['tweet_created'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "URJsJAIyOkDf"
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 516
        },
        "id": "huIvIJSKOkDf",
        "outputId": "5a673f76-39bf-4e74-9289-252d7aa3ae02"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Sentiment Analysis Over Time')"
            ]
          },
          "execution_count": 77,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsMAAAHiCAYAAAANlMFMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZgU5bn38d+NbKIosogKIkQJyuaggxpZRE0cXCG4oomiMby+R02ixmjipfGcxByN5hhcEl5PJMSjASJKNAaVEFRQEA8qyKqiQcANxAURle1+/6iacbqZme6ZqZme6uf7ua65mK6qrnr6oafrV0/dVW3uLgAAACBEzQrdAAAAAKBQCMMAAAAIFmEYAAAAwSIMAwAAIFiEYQAAAASLMAwAAIBgEYYBpI6ZjTez6wvdjsZmZjea2f31XEfR9Z2ZDTGzVwvdDgDpRBgGkAgzG2xmc83sEzP70MyeM7OBCax3jJk9W3mau1/i7r+o77rr0Ja8w6iZPW1mH5lZq4ZuV23Ute8scrWZvW5mn5vZajP7z8Z4fWZ2npltin8+N7MdlR5vcvc57t6rodsBoDgRhgHUm5ntIekxSXdKai+pi6R/l/RlIdtVKGbWXdIQSS7ptII2Jjl3SBor6XxJbSWdKOl4SX9JekNm1rzyY3d/wN13d/fd4+2+U/44ngYAdUYYBpCEr0uSu09y9+3u/rm7z3D3V8oXMLOLzGx5PFr6pJkdUGmem9kl8ajjx2Z2dzwSeYik8ZK+EY8CfhwvP9HMfhn/PszM1prZT8xsnZm9a2YjzewkM3stHqX+WaVtNTOza83sDTPbYGZ/MbP28bzucVsuiEc+PzCz6+J5wyX9TNLZcVsW1dAf50t6XtJESRdUnhG3/W4z+7uZfWpm883swErzx5nZGjPbaGYvmtmQqjYQP//yrGmvmNm34767Pe6PjWa22Mz6VtF3Hc3ssbjPPzSzOWa2037BzHpK+jdJ57n7PHff5u5LJZ0uabiZHWdmR5rZe2a2S6XnfdvMXqlFv3/PzFZLmlVD31bVF8PMbG2lx6viUexXzOwzM7vXzDqb2eNxn880s70qLX+URWc1PjazRWY2rDbbB5BuhGEASXhN0nYz+5OZnVg5aEiSmY1QFCRHSeokaY6kSVnrOEXSQEn9JZ0lqczdl0u6RNK8eBSwXTXb30dSa0Uj0jdI+m9J35F0uKIR2uvNrEe87OWSRko6RtJ+kj6SdHfW+gZL6qVo5PMGMzvE3Z+Q9CtJU+K2HFpDf5wv6YH4p8zMOmfNP0fRyPleklZKuqnSvP+VVKJohP3Pkh40s9ZVbONP8WuUJJnZofHr/7ukEyQNVXSQsqei/txQxTqukrRW0f9JZ0X/R17FcsdLWuvuL1Se6O5rFIX+b7n7fEmfSTqu0iLnxq9Byq/fj5F0iKSyKtpQW6dL+paiPjhV0uOKXl8nRfu+H0iSmZX32S8V9fmPJT1kZp0SaAOAFCAMA6g3d9+oKEC6oiC63swerRQCL5H0n+6+3N23KQqVJZVHhyXd7O4fu/tqSU8pCoT52irpJnffKmmypI6Sxrn7p/EI5jJJ5eH1EknXuftad/9S0o2Szsg6Nf/v8ej2IkmLKj03JzMbLOkASX9x9xclvaEoFFY2zd1fiPvigcqv1d3vd/cN8ejrbyS1UhTMsz0q6evxqK0kfVdRUN8S90dbSQdLsrjf361iHVsl7SvpAHffGtfeVhWGO0qq6vmKp3eMf58kaXTcD20lnaSvDnry6fcb3f0zd/+8mm3Vxp3u/r67v63o4Gu+u7/s7l9ImiZpQLzcdyRNd/fp7r7D3f8haUHcdgABIAwDSEQcuMa4e1dJfRWN/v02nn2ApHHxaeiPJX0oyRSNZJZ7r9LvmyXVphZ0g7tvj38vD1LvV5r/eaX1HSBpWqW2LJe0XdHIaBJtuUDSDHf/IH78Z2WVStS0fjP7sUXlJJ/E7dtTX4XNCnGomyLpO3Fpw2hJ/xPPmyXpLkUjr+vM7B6L6rqz3apoZHqGmb1pZtdW85o+UBSaq7JvPL/8tY6y6KK6UZJecve34nn59PuaarZRF9n//zW9H84sb1fctsGq/vUCKDKEYQCJc/cViupl+8aT1kj6P+7ertLPru4+N5/VJdy8NZJOzGpL63gEsV5tMbNdFZUkHBPXz74n6QpJh8ZlDDWK64N/Eq9jr7gs5BNFBw5V+ZOk8xSVMWx293kVDXW/w90Pl9RbUanA1Tu9mGjk/Cp3/5qiC/2uNLPjq9jOLEn7m9kRWe3dX9JRkv4Zr2+ZpLcUXeRWuURCyq/fk/6/zscaSf+T1a7d3P3mArQFQAEQhgHUm5kdbGZXmVnX+PH+ikYqn48XGS/pp2bWJ56/p5mdmefq35fU1cxaJtTc8ZJuKi/RMLNOcU1zvm3pXtVFZrGRikY7eysqfShRVAM7R1EdcS5tJW2TtF5SczO7QVJVI7qSpDj87pD0G8WjwpJkZgPjC9paKKrj/SJeLoOZnWJmB5mZKQrd26tazt1fU9RvD8QXm+0S/18+JGmmu8+stPifJf1QUc3yg5Wm16ffG9L9kk41s7L4dbWOL8jrWuiGAWgchGEASfhU0pGS5pvZZ4pC8BJFF2jJ3adJukXSZDPbGM87Mc91z5K0VNJ7ZvZBroXzME5Rve0MM/s0buuReT63PNxtMLOXqph/gaQ/uvtqd3+v/EdRycJ5WfWxVXlS0hOKLkh8S1GIzVU6cJ+kfopCXbk9FNVufxSvZ4OikohsPSXNlLRJ0jxJv3P3p6rZzmWS/hBvZ1PczqcVXahW2SRFF8LNqlQqItWv3xtMfBFg+QWe6xX199Vi/wgEw6q+VgIAkAZmdr6kse4+uNBtAYA04sgXAFLKzNoouv/vPYVuCwCkFWEYAFLIzMoUndZ/X5kXqgEAaoEyCQAAAASLkWEAAAAEizAMAACAYOW6zU+D6dixo3fv3r1QmwcAAEAgXnzxxQ/cvVNV8woWhrt3764FCxYUavMAAAAIhJm9Vd08yiQAAAAQLMIwAAAAgkUYBgAAQLAKVjNcla1bt2rt2rX64osvCt0UNKDWrVura9euatGiRaGbAgAAAtekwvDatWvVtm1bde/eXWZW6OagAbi7NmzYoLVr16pHjx6Fbg4AAAhckyqT+OKLL9ShQweCcBEzM3Xo0IHRfwAA0CQ0qTAsiSAcAP6PAQBAU9HkwnCh7bLLLiopKVHfvn115plnavPmzZKiEo4RI0aoZ8+eOvDAA/XDH/5QW7ZskSRt3rxZ5513nvr166e+fftq8ODB2rRpkyRp99131+LFi1VSUqKSkhK1b99ePXr0UElJib75zW9q1apV6tu3rzZv3qwOHTpo48aNGe0ZOXKkpkyZookTJ6pTp04V6ykpKdGyZcsqltuwYUPF9H322UddunSpeFzeziQsXLhQ06dPT2x9AAAAhUQYzrLrrrtq4cKFWrJkiVq2bKnx48fL3TVq1CiNHDlSr7/+ul577TVt2rRJ1113nSRp3Lhx6ty5sxYvXqwlS5bo3nvvzbg4rF+/flq4cKEWLlyo0047TbfeeqsWLlyomTNnVizTpk0blZWVadq0aRXTPvnkEz377LM69dRTJUlnn312xXoWLlyo3r17VyzboUOHiumXXHKJrrjiiorHLVu2TKx/CMMAAKCYEIZrMGTIEK1cuVKzZs1S69atdeGFF0qKRo9vv/12TZgwQZs3b9a7776rLl26VDyvV69eatWqVa23N3r0aE2ePLni8bRp01RWVqY2bdrUqf3r1q3T4YcfLklatGiRzEyrV6+WJB144IHavHmz1q9fr9NPP10DBw7UwIED9dxzz0mSPvvsM1100UU64ogjNGDAAD3yyCPasmWLbrjhBk2ZMkUlJSWaMmWKnnnmmYoR6AEDBujTTz+tU1sBAAAKoUndTaKyf//bUi17Z2PuBWuh93576Oen9slr2W3btunxxx/X8OHDtXTp0opQWW6PPfZQt27dtHLlSl100UU64YQTNHXqVB1//PG64IIL1LNnz1q3r6ysTBdffLE2bNigDh06aPLkybrssssq5k+ZMkXPPvtsxeN58+Zp1113rXZ9e++9t7744gtt3LhRc+bMUWlpqebMmaPBgwdr7733Vps2bXTxxRfriiuu0ODBg7V69WqVlZVp+fLluummm3TcccdpwoQJ+vjjj3XEEUfom9/8pv7jP/5DCxYs0F133SVJOvXUU3X33Xdr0KBB2rRpk1q3bl3r1w0AAFAoTTYMF8rnn3+ukpISSdHI8Pe+9z2NHz++xueUlJTozTff1IwZMzRz5kwNHDhQ8+bN0yGHHFKrbbds2VKnnXaapk6dqtNPP10vv/yyysrKKuafffbZFSE0X0cffbSee+45zZ49Wz/72c/0xBNPyN01ZMgQSdLMmTMzao83btyoTZs2acaMGXr00Ud12223SYru9FE+qlzZoEGDdOWVV+q8887TqFGj1LVr11q1DwAAoJCabBjOdwQ3aeU1w5X17t1bU6dOzZi2ceNGrV69WgcddJCk6EK5UaNGadSoUWrWrJmmT59e6zAsRaUSv/jFL+TuGjFiRL2/mGLo0KGaM2eO3nrrLY0YMUK33HKLzEwnn3yyJGnHjh16/vnndxrRdXc99NBD6tWrV8b0+fPnZzy+9tprdfLJJ2v69OkaNGiQnnzySR188MH1ajMAAEBjoWY4D8cff7w2b96s++67T5K0fft2XXXVVRozZozatGmj5557Th999JEkacuWLVq2bJkOOOCAOm1r2LBhev3113X33Xdr9OjR9W77kCFDdP/996tnz55q1qyZ2rdvr+nTp2vw4MGSpBNOOEF33nlnxfLlBwJlZWW688475e6SpJdfflmS1LZt24y64DfeeEP9+vXTNddco4EDB2rFihX1bjMAAEBjyRmGzWx/M3vKzJaZ2VIz+2EVy5iZ3WFmK83sFTM7rGGaWxhmpmnTpunBBx9Uz5499fWvf12tW7fWr371K0lRIDzmmGPUr18/DRgwQKWlpTr99NPrtK1mzZrpjDPO0IYNG3TMMcdkzCu/cK38Z+7cuTnX1717d7m7hg4dKkkaPHiw2rVrp7322kuSdMcdd2jBggXq37+/evfuXVEScv3112vr1q3q37+/+vTpo+uvv16SdOyxx2rZsmUVF9D99re/Vd++fdW/f3+1aNFCJ554Yp1eNwAAQCFY+chftQuY7StpX3d/yczaSnpR0kh3X1ZpmZMkXS7pJElHShrn7kfWtN7S0lJfsGBBxrTly5fXqbQA6cP/NQAAaCxm9qK7l1Y1L+fIsLu/6+4vxb9/Kmm5pC5Zi42QdJ9HnpfULg7RAAAAQJNVq5phM+suaYCk+VmzukhaU+nxWu0cmIHGtflD6cY9pRf/VOiWoJjduKf0+LXJr/e+EdJ/dtt5+rgSafzg5LeXJncdEf009XWG6M2no7+JdcsL3ZLqvfDfURu/qOb2rVs/j+Y/d0fN65l2SbRcyP7n29Kv0h/38g7DZra7pIck/cjd63QDYDMba2YLzGzB+vXr67IKIH8fvxX9u+DewrYDxW/+75Nf55tPS19+svP0j/4lvbc4+e2lyQevRj9NfZ0hWvZI9O9bzxW2HTWZH98uddP7Vc//PLogXs//rub1LJqUXJvS6o1Z0pZNhW5FveUVhs2shaIg/IC7P1zFIm9L2r/S467xtAzufo+7l7p7aadOnerSXgAAACAx+dxNwiTdK2m5u/9XNYs9Kun8+K4SR0n6xN3fTbCdQN3luEgUABCgXPsG9h3ByOdLNwZJ+q6kxWZW/m0UP5PUTZLcfbyk6YruJLFS0mZJFybfVKC2rNANAAA0OewbkCmfu0k86+7m7v3dvST+me7u4+MgrPguEpe6+4Hu3s/dF+Rab1NlZrrqqqsqHt9222268cYbJUk33nijunTpknGv348//liS9MILL2jYsGHq2bOnDjvsMJ188slavDizrq+kpETnnHOOJOmPf/xjxTpatmypfv36qaSkRNdee60mTpyoyy67TM8884y+8Y1vZKxj27Zt6ty5s9555x2NGTNGPXr0qFjP0UcfnbHsk08+WTFv9913V69evVRSUqLzzz8/0T7761//mvGVzgAApJ4RmkPRZL+OuVBatWqlhx9+WD/96U/VsWPHneZfccUV+vGPf5wx7f3339dZZ52lP//5zxWB9Nlnn634djYpuq/u9u3bNWfOHH322We68MILdeGF0QB69+7d9dRTT1Vsb+LEiZKib49bu3at3nrrrYpvtJs5c6b69Omj/fbbT5J066236owzzqjytZSVlamsrExS9M12t912m0pLq7zFXr389a9/1SmnnKLevXsnvu5kcKoLAJCNMglE+DrmLM2bN9fYsWN1++235/2cu+66SxdccEHGyOzgwYM1cuTIiseTJk3Sd7/7XZ1wwgl65JFH8lpvs2bNdNZZZ2ny5MkV0yZPnlyvr2l+8MEHdeWVV0qSxo0bp6997WuSpDfffFODBg2SJL344os65phjdPjhh6usrEzvvhuVf7/xxhsaPny4Dj/8cA0ZMkQrVqzQ3Llz9eijj+rqq69WSUmJ3njjDd1xxx3q3bu3+vfvXzESXhAc1QMAsuXcN7DvCE3THRl+/Nrkbx+0Tz/pxJtzLnbppZeqf//++slPfrLTvNtvv13333+/JGmvvfbSU089paVLl+qCCy6ocZ1TpkzRP/7xD61YsUJ33nmnzj333LyaPHr0aH3/+9/XNddcoy+//FLTp0/Xf/3XV9cxXn311frlL38pSerTp48eeOCBGtc3ZMgQ/frXv5YkzZkzRx06dNDbb7+tOXPmaOjQodq6dasuv/xyPfLII+rUqZOmTJmi6667ThMmTNDYsWM1fvx49ezZU/Pnz9e//du/adasWTrttNN0yimnVIxQ33zzzfrXv/6lVq1aVZSRAACQDowIh6bphuEC2mOPPXT++efrjjvu0K677poxr6oyiWxHHnmkNm7cqBNOOEHjxo3TggUL1LFjR3Xr1k1dunTRRRddpA8//FDt27fP2ZbS0lJt2rRJr776qpYvX64jjzwy43k1lUlUZZ999tGmTZv06aefas2aNTr33HM1e/ZszZkzR6NGjdKrr76qJUuW6Fvf+pYkafv27dp33321adMmzZ07V2eeeWbFur788ssqt9G/f3+dd955GjlyZMboeMHwuQYAyEYZBGJNNwznMYLbkH70ox/psMMOq6jrrUmfPn300ksvacSIEZKk+fPna+rUqXrsscckRSUSK1asUPfu3SVJGzdu1EMPPaTvf//7ebVl9OjRmjx5spYvX16vEolyRx99tP74xz+qV69eGjJkiCZMmKB58+bpN7/5jVavXq0+ffpo3rx5Gc/ZuHGj2rVrp4ULF1az1q/8/e9/1+zZs/W3v/1NN910kxYvXqzmzQvxVuNUFwAgG2USyETNcDXat2+vs846S/fem/vbyy699FJNnDhRc+fOrZi2efNmSdKOHTv0l7/8RYsXL9aqVau0atUqPfLII5o0Kf9vrhk9erTuv/9+zZo1qyJw18eQIUN02223aejQoRowYICeeuoptWrVSnvuuad69eql9evXV4ThrVu3aunSpdpjjz3Uo0cPPfjgg5Ikd9eiRYskSW3bttWnn35a8XrXrFmjY489Vrfccos++eQTbdqU/m+nAQAAxYkwXIOrrrpKH3zwQca022+/PePWaqtWrdI+++yjKVOm6Kc//akOOuggHX300Zo6daouu+wyzZkzR126dKm4+4MkDR06VMuWLau4MC2XQw45RLvttpuOO+447bbbbhnzyi9cK//ZsmVLzvUNGTJEa9as0dChQ7XLLrto//331+DBgyVJLVu21NSpU3XNNdfo0EMPVUlJSUXIf+CBB3Tvvffq0EMPVZ8+fSouBDznnHN06623asCAAXr99df1ne98R/369dOAAQP0gx/8QO3atcvrdTYcToUBALKxb0Ck6ZZJFEjlUczOnTtXjPBK0X2Gy+85nO2oo47SM888U+W8559/PuPxLrvsovfee6/i8apVqzLmjxkzRmPGjMmYVlV5Qvkt2PLx9NNPV/x+4IEHyivVSs2YMSNj2ZKSEs2ePXundfTo0UNPPPHETtMHDRqUcZ/hZ599Nu92NSjuJgEAyMa+AVkYGUbx4yIJAEC2nPsG9h2hIAyjiHH0DwAF0aQHIXLsGxg5Dg5hGAAAJKQIgmSTDvJoCE0uDDtvwqLX+P/HvKcAANnYNyDSpMJw69attWHDBgJxEXN3bdiwQa1bt274jXGqCwCQLde+gX1HcJrU3SS6du2qtWvXav369YVuChpQ69at1bVr10I3AwCAnTEgF5wmFYZbtGihHj16FLoZKDZ8sAEAsrFvQKxJlUkAyeJUFwAgG2USyEQYBgAAQLAIwwgAp8IAANnYNyBCGEbx4lQXACAb+wZkIQwDAAAgWIRhFD+uGAYAZMu1b2DfEQzCMIoYp8IAANnYNyATYRgAACAbtcXBIAwjAJzqAgBko0wCEcIwihdH9QDQuNLwuZuziSl4DUgUYRjFj6N7AEA29g2IEYZRxDi6BwBkY9+ATIRhAAAABIswjABwKgwAkC3XvoF9RygIwyheabiQAwDQuNg3IAthGAAAYCeE5lAQhlH8uGIYAJAt576BfUcoCMMoYhzVAwCysW9AJsIwAAAAgkUYRgA41QUAyMa+ARHCMIoXVwwDALKxb0AWwjAAAEAFRoxDQxhG8eNuEgCAbOwaECMMAwCAZDXpQQjKJJCJMAwAABJC0ET6EIYRgKY8QgEAKIxq9g1NelQbDYEwjOLFFcMAgGzsG5CFMIzix1E+ACBbrn0D+45gEIYBAAAQLMIwihinwgAAtcWIcGgIwwgAH2wAAKBqhGEULy6SAADshH0DMhGGAQAAECzCMIofVRIAgJ1wn2FECMMoYpwKAwBkoYQOWQjDAAAACBZhGAHglBcAIEvOcgj2HaEgDKN4cSoMALCTXPsGQnBoCMMAACBhxRAoGVAJBWEYxY8rgwGgcaTqjBxlEogQhlHE0vShDABoFLkCOwMowSEMAwAAIFiEYQSAo3wAQBZGgBEjDKN4pap2DQDQONg3IBNhGMWPo38AwE6q2zewzwgNYRgAAADBIgyjiHEqDACQhRI6ZCEMIwCc8gIAZMlVQkeJXTAIwyheHP0DAHbCfYaRiTAMAACQjQGVYBCGUfw4ygcA7IQyCUQIwyhiHNUDALIw4osshGEAAJCsVI+qprntqAvCMALABxsANI4UjbqmOrAjSYRhAAAQkBQFdjSKnGHYzCaY2TozW1LN/D3N7G9mtsjMlprZhck3EwAAAEhePiPDEyUNr2H+pZKWufuhkoZJ+o2Ztax/04CEcCoMALCTavYN7DOCkzMMu/tsSR/WtIiktmZmknaPl92WTPMAAAASxN0kkKV5Auu4S9Kjkt6R1FbS2e6+I4H1AgAAAA0qiQvoyiQtlLSfpBJJd5nZHlUtaGZjzWyBmS1Yv359ApsG8sEpLwBAlpzlEOw7QpFEGL5Q0sMeWSnpX5IOrmpBd7/H3UvdvbRTp04JbBoAACBJhODQJBGGV0s6XpLMrLOkXpLeTGC9AAAABUJtcShy1gyb2SRFd4noaGZrJf1cUgtJcvfxkn4haaKZLVb0zrnG3T9osBYDtcWVwQCAnVAmgUjOMOzuo3PMf0fSCYm1CAAAoKHkupsEAyjB4RvoEAA+2AAAWQi9iBGGAQAAECzCMAAACAgXxiETYRjFj1NhANDI0vC5m4Y2ojEQhgEAQDLS8FXHaWgjGhVhGAAAAMEiDCMAnAoDAGTJVUJHiV0wCMMAACAg3GcYmQjDAAAA2agtDgZhGMWPo3wAwE4ok0CEMAwAAMLBiC+yEIYBAAAqMCIcGsIwAsAHGwAgC7sGxAjDAAAgIJRJIBNhGAAABIQhYWQiDKP48bkHAMgXd5EIDmEYAAAEhDIJZCIMIwAc5QNAo0rF6GquNqbhNSAJhGEAAAAEizAMAAASkoIShJxfusGIcGgIwyh+qThdBwBoVOwbECMMAwCAgKRg9BqNijAMAABQjhHj4BCGEQA+2AAA2dg3IEIYBgAA4ch5AR1CQxgGAABAsAjDKH7UfwEAslW7b2CfERrCMAAACAhlEshEGAYAAECwCMMIAKe8AADZcuwb2HUEgzAMAADCketuElxnEhzCMAAAQDZKi4NBGEbx4ygfABpZCj53c+0bUvASkAzCMAAASEYqvtAiDW1EYyIMIwAc3gMAsnGfYUQIwwAAAAgWYRgAAIQjFaUcaEyEYRQ/LqADAGTLuW9g3xEKwjAAAAgI9xlGJsIwAADATiinCAVhGAHgKB8AkI0yCUQIwwAAIBxcQIcshGEAAIAKjAiHhjCM4sfFEACAbOwbECMMAwAAIFiEYQAAgHKMGAeHMIwA8MEGAMjGvgERwjCKGB90AFAQTXl0lbtJIAthGAHggw8AGgeft0gfwjAC0IRHKAAAhVHt6LXnmI9iQxgGAAABYfQamQjDKH4c3QMAdsK+ARHCMAAAAIJFGAYAAOHIdTcJziYGhzCMAPDBBgDIQuhFjDAMAACAYBGGAQBAQLibBDIRhlH8OBUGANhJjvsMIxiEYQAAEA6+jhlZCMMAAAAIFmEYAeCUFwAgS84SOvYdoSAMAwCAhDXlIMl9hpGJMAwAALATaotDQRgGAADJKKqL0xghDgVhGAAAAMEiDAMAAFRgRDg0hGEAAAAEizAMAADCUVR1zUgCYRgAAISnuluoeY75KDqEYQAAAAQrZxg2swlmts7MltSwzDAzW2hmS83smWSbCAAAkJQ8yyQopwhGPiPDEyUNr26mmbWT9DtJp7l7H0lnJtM0AACAhpKjDIIyiWDkDMPuPlvShzUscq6kh919dbz8uoTaBgAA0MgIwaFJomb465L2MrOnzexFMzs/gXUCAAAkj/IHZGme0DoOl3S8pF0lzTOz5939tewFzWyspLGS1K1btwQ2DQAAUAeUQSCWxMjwWklPuvtn7v6BpNmSDq1qQXe/x91L3b20U6dOCWwaAACgNhgZRqYkwvAjkgabWejQgtIAABRQSURBVHMzayPpSEnLE1gvAABIozSPuqa57aiTnGUSZjZJ0jBJHc1sraSfS2ohSe4+3t2Xm9kTkl6RtEPSH9y92tuwAQAAFB6hF5GcYdjdR+exzK2Sbk2kRQAAAA2FC+iQhW+gAwAAQLAIwwAAIDzV1gZTPhEawjAAAAgIZRLIRBgGAABAsAjDAAAgQLnKISiXCAVhGAAAhCPX3SS4z3BwCMMAAAA7obY4FIRhAAAQnpwjwIwQh4IwDAAAAsKILzIRhgEAQIC4zzAihGEAAAAEizAMAADCketuEggOYRgAACQsBaUG1V1AVz6dW6wFgzAMAACSwagrUogwDAAAApJnYCfYB4MwDAAAApSjDIIyiWAQhgEAQDhyDvgSgkNDGAYAAECwCMMAACA8lEEgRhgGAAAB4cI4ZCIMAwAAlGPEODiEYQAAECBCLyKEYQAAEA7uH5y8lI+mE4YBAAAQLMIwAAAIT7Wjmeke5UTtEYYBAEBAKJNAJsIwAABIVipqSHO1MQ2vAUkgDAMAAKDuUnHwUz3CMAAASEgKShBy3U0i5cEOtUcYBgAA4SH0IkYYBgAAQLAIwwAAICApKOVInXSPshOGAQBAgLjPMCKEYQAAkBCCJNKHMAwAAIBgEYYBAECyct2+rCkrv8sEd5sIBmEYAACEI81BvalK+YEDYRgAACAboTkYhGEAABCeXKOZKR/tRP4IwwAAICC5RnwJwaEhDAMAAKAe0n0AQRgGAADJosQAKUIYBgAACUnBRWdcGIcshGEAABCe6kavGdUODmEYAAAAwSIMAwCAgFAmkbiUj6YThgEAQIByBbh0BzzkjzAMAABQgRAcGsIwAAAIR953k6CcIhSEYQAAEJ6cda6MEOcv3X1FGAYAAECwCMMAACAgOcofUn5nBNQeYRgAAASI0IsIYRgAAISDr2NOXspH0wnDAAAACBZhGAAAJCwFI4XVjmamoO1IFGEYAAAEhDIJZCIMAwCAZFCPixQiDAMAgABVUw5RPjnlF4U1rnT3FWEYAACEg9FrZCEMAwCA8DDyixhhGAAAAMEiDAMAgIDkKpNgxLjWUj7KThgGAAABSneAQ3IIwwAAAAgWYRgAAISDu0kgC2EYAACEp7o615TXv6L2CMMAAACoh3QfQBCGAQAAECzCMAAACFCu0cx0j3YifznDsJlNMLN1ZrYkx3IDzWybmZ2RXPMAAEDqNOW625wX0DXhtqNB5DMyPFHS8JoWMLNdJN0iaUYCbQIAAKlUTHdqKKbX0sCa8sFPHnKGYXefLenDHItdLukhSeuSaBQAAECDyhng0h3wkL961wybWRdJ35b0+/o3BwAAoCEx4otMSVxA91tJ17j7jlwLmtlYM1tgZgvWr1+fwKYBAAASlPJT/qi95gmso1TSZIsK0jtKOsnMtrn7X7MXdPd7JN0jSaWlpbzbAABAgRBDEKl3GHb3HuW/m9lESY9VFYQBAAAKjq9jbgDpPrDIGYbNbJKkYZI6mtlaST+X1EKS3H18g7YOAACgIeQqh6BcIhg5w7C7j853Ze4+pl6tAQAAKChCcGj4BjoAABCQPMskKKcIBmEYAAAEiDKJxKS8rwjDAAAA5VIe7FB7hGEAABAOyh+QhTAMAADCwcgvshCGAQAAUA/pPsAgDAMAgIQ14XCUs0yiCbcdDYIwDAAAkpGmelzKJRAjDAMAgICkKLCjURCGAQAAECzCMAAACFA1ZRIV5ROUUeQt5SUnhGEAABCONNU1o1EQhgEAAHZCaA4FYRgAAIQn55n9dJ/6R/4IwwAAICDcZzh56e4zwjAAAMBOKJMIBWEYAAAEKNdoZrpHO5E/wjAAAAgHd5NAFsIwAAAIT3X3xk35PXNRe4RhAAAA1F3KDyAIwyheKf/jBIDUatKfv5RJIBNhGAAABChHYG/SgR5JIgwDAICEFMOoKyE4NIRhAAAQjnzvJsFdJ4JBGAYAAOHJVQZBmUQwCMMAAADlCMHBIQwDAICAUP6ATIRhAAAQIEaAESEMAwCAcHBhXPJSXlpCGAYAAKiQ7mCH2iMMAwCA8KR8NBPJIQwDAICAUCaBTIRhAAAA1EO6R9kJwwAAIEDVBLiK8ol0BzzkjzAMAADCwd0kkIUwDAAAEpaCUVUuoEOMMAwAAJLBqCtSiDAMAAACkiuwM2JcaykfZScMAwCAAKU7wCE5hGEAAAAEizAMAADCQVkzshCGAQBAeKqrc015/WthpLvPCMMAAAAIFmEYAAAEhDoJZCIMAwCAAKX71D6SQxgGAADh4ItBkpfyOmvCMAAAAIJFGAYAAOFJ+WgmkkMYBgAAAclRJkFIDg5hGAAAJIs8iRQhDAMAgACR2JOT7r4kDAMAgISk4E4N3E0CWQjDAAAgPNXWBqd7lBO1RxgGAABAsAjDAAAAqLuU34GDMAwAAIBgEYYBAADKpXyUE7VHGAYAAAHhbhLIRBgGAAABYgQYEcIwAAAA6iHdBxaEYQAAEI6cX7qR7mCH2iMMAwCA8HChHGKEYQAAAASLMAwAAALC3SQSl/JRdsIwAABIWIrDUcqDHWqPMAwAAMKR8wI6hIYwDAAAkkHQRAoRhgEAQHiqLYegTCI0hGEAAADUQ7oPIAjDAAAgQOkOcEhOzjBsZhPMbJ2ZLalm/nlm9oqZLTazuWZ2aPLNBAAAAJKXz8jwREnDa5j/L0nHuHs/Sb+QdE8C7QIAAGh83FotOM1zLeDus82sew3z51Z6+LykrvVvFgAAQAMi9CYn5X2ZdM3w9yQ9nvA6AQAAgAaRc2Q4X2Z2rKIwPLiGZcZKGitJ3bp1S2rTAAAAQJ0kMjJsZv0l/UHSCHffUN1y7n6Pu5e6e2mnTp2S2DQAAEAdcJ9hROodhs2sm6SHJX3X3V+rf5MAAACQHuk+gMhZJmFmkyQNk9TRzNZK+rmkFpLk7uMl3SCpg6TfWfQ1jNvcvbShGgwAAAAkJZ+7SYzOMf9iSRcn1iIAAICGlvI7ICA5fAMdAABIVpqDZprbjjohDAMAgIRYoRsA1BphGAAABIgR4MSkfDSdMAwAAIBgEYYBAAAqpHuUE7VHGAYAAOFJ+al9JIcwDAAAgHpI94EFYRgAAAQo3QEOySEMAwAAlKN8IjiEYQAAAASLMAwAAMLDCDBihGEAAIAKhORaS/mBBWEYAAAAwSIMAwCAAKV7NBPJIQwDAICEETSRHoRhAACAcimvfy2MdPcZYRgA0oKdNJo6s0K3IH/8PSFGGAYAAECwCMMAAAAIFmEYAAAEqLoyCconQkMYBgAAQN2lvP6aMAwAaZHyHQ4ANEWEYQAAEB4OLhEjDAMAgHDkCsGE5OAQhgEAABAswjAAAACCRRgGAABAsAjDAJAa1DICyeE+w4gQhgEAAFB3Kb/okDAMAACSlfJwhLAQhgEAQHgI7IgRhgEAQEKs0A2oP0JycAjDAAAACBZhGADqqrFHkBixAhLE31Ny0t2XhGEAAAAEizAMAACAYBGGAQBAeNJ9Zh8JIgwDAACg7lJ+PQNhGABSI907HCAVUh7sUHuEYQAAECBCLyKEYQAAAASLMAwAAIBgEYYBAEB4qq0Npnyi9tLdZ4RhAAAABIswDABpwVXuSI00vFfT0EY0BsIwAABIhlmhWwDUGmEYAACgHGdgai/lfUYYBgAA4Ul5gENyCMMAAAAIFmEYAAAAwSIMA0BqcFoXSA73GUaEMAwAAIB6SPcBBGEYAAAAwSIMAwCA8HA3CcQIwwAAAOUIycEhDAMAAKDuUn4AQRgGgLRI+Q4HaFr4e0KEMAwAAFCBkBwawjAAAEgWZzGQIoRhAAAQHgJ7gtLdl4RhAACQECt0A4BaIwwDQGqke/QFaFqq+XtixDg4hGEAAAAEizAMAACAYBGGAQBAeCiHSE7K+5IwDAAAUCHdwQ61RxgGAABAsAjDAJAWKT8VCTQt/D0hQhgGAABAPaT7wIIwDAAAUI4zMMHJGYbNbIKZrTOzJdXMNzO7w8xWmtkrZnZY8s0EAABIEKEXsXxGhidKGl7D/BMl9Yx/xkr6ff2bBQAAADS8nGHY3WdL+rCGRUZIus8jz0tqZ2b7JtVAAAAAoKGY53GawMy6S3rM3ftWMe8xSTe7+7Px439KusbdF9S0ztLSUl+woMZFkvfZB9LDY3MutnL9Jm36clsjNAgNqYVvUZ8tiyVJC1sdXuDWoBiZuw7d8pKk5N9jJV++uNN6m/kO9d/ycoNsL02q6pumuM4Qlfej1HT7cu9t72m/7W9rs7XRay0PqXa+VPNr4D3zVR+82uIQfd6sTc7lW+26uw750aMN3awqmdmL7l5a1bzmjdyQsYpKKdStW7fG3HTEd0hffJJzsdbbP5fv2N4IDUJj2GxttNuOzwrdDBS5hnqPVbde3tMN0wf0a/2sad5N+29brbXNuzXZvvysWVtpu/RO865VtvGzZm31oXfQ+l32zus1NNXX2Rg+araX9trxkZrJ8+qHXZpotEpiZPj/SXra3SfFj1+VNMzd361pnQUZGQYAAEBwahoZTuLWao9KOj++q8RRkj7JFYQBAACApiBnmYSZTZI0TFJHM1sr6eeSWkiSu4+XNF3SSZJWStos6cKGaiwAAACQpJxh2N1H55jvki5NrEUAAABAI+Eb6AAAABAswjAAAACCRRgGAABAsAjDAAAACBZhGAAAAMEiDAMAACBYhGEAAAAEizAMAACAYBGGAQAAECzCMAAAAIJFGAYAAECwCMMAAAAIFmEYAAAAwSIMAwAAIFiEYQAAAATL3L0wGzZbL+mtgmxc6ijpgwJtu1jQh8mgH+uPPqw/+rD+6MNk0I/1Rx9W7QB371TVjIKF4UIyswXuXlrodqQZfZgM+rH+6MP6ow/rjz5MBv1Yf/Rh7VEmAQAAgGARhgEAABCsUMPwPYVuQBGgD5NBP9YffVh/9GH90YfJoB/rjz6spSBrhgEAAAAp3JFhAAAAoGmEYTPb38yeMrNlZrbUzH4YT29vZv8ws9fjf/eKpx9sZvPM7Esz+3HWulaZ2WIzW2hmC2rY5nAze9XMVprZtZWmPxBPX2JmE8ysRTXPvyx+rptZx0rTr463vTBex3Yza1/fPsqlyPpwTzP7m5ktil/LhfXtn3wUWR/uZWbTzOwVM3vBzPrWt3/yldJ+rHI5i9wRr/cVMzssiT7Kpcj6sNq2NaQi68Pz4vffYjOba2aHJtFHuRRZH46I+3ChmS0ws8FJ9FE+iqkfK80faGbbzOyM+vRNk+HuBf+RtK+kw+Lf20p6TVJvSb+WdG08/VpJt8S/7y1poKSbJP04a12rJHXMsb1dJL0h6WuSWkpaJKl3PO8kSRb/TJL0f6tZxwBJ3WvanqRTJc2iD2vXh5J+VqmdnSR9KKklfVirPrxV0s/j3w+W9M/GeB+muB+rXC6e/ng8/ShJ8+nDWvdhtW2jD/Puw6Ml7RX/fiLvwzr14e76qjS0v6QVvBdr34+V1j9L0nRJZzRWPzbkT5MYGXb3d939pfj3TyUtl9RF0ghJf4oX+5OkkfEy69z9fyVtreMmj5C00t3fdPctkibH25K7T/eYpBckda2mzS+7+6oc2xmt6E3U4IqsD11SWzMzRR9gH0raVsd25q3I+rC3og8rufsKSd3NrHMd21krKe3H6pYbIem+eNbzktqZ2b51bGfeiqkPE2hbnRRZH85194/ixZ6v7vlJK7I+3BRPk6TdFO1nGkUx9WPsckkPSVpXx/Y1OU0iDFdmZt0VjXbNl9TZ3d+NZ70nKZ+duUuaYWYvmtnYapbpImlNpcdr42mV29FC0nclPZF34zOf30bScEVvmEZVBH14l6RDJL0jabGkH7r7jlquo16KoA8XSRoVr+MISQeokXaglaWtH6tYLue6G1oR9GHBFVkffk/R2YpGVQx9aGbfNrMVkv4u6aI82py4tPejmXWR9G1Jv8+jranRvNANqMzMdlcUHn/k7hujgcGIu7uZ5XMkN9jd3zazvSX9w8xWuPvsOjTnd5Jmu/ucOjxXikoknnP3D+v4/Dopkj4sk7RQ0nGSDozbMMfdN9ahDbVWJH14s6RxZrZQ0QHFy5K212H7dZbSfqzv332i6MP6K6Y+NLNjFYXhRqt3jbdbFH3o7tMkTTOzoZJ+Iembddh+nRVJP/5W0jXuvqNy+9OuyYwMx0cfD0l6wN0fjie/X35KMv4355C8u78d/7tO0jRJR1hUvF5+Udslkt6WtH+lp3WNp5W35eeKalWvrDTtyfj5f8jzJZ2jRiqRKFdEfXihpIfjMzQrJf1LUd1rgyuWPnT3je5+obuXSDo/Xs+budqdlDT2Y1XL5Vp3QyqiPiyYYupDM+sv6Q+SRrj7hnz7oL6KqQ8rtWW2pK9ZpYuOG1oR9WOppMlmtkrSGZJ+Z2Yj8+yGpsubQOGyogLt+yT9Nmv6rcosLv911vwbVam4XFEdUNtKv8+VNLyK7TVXFAx66Kvi8j7xvIvj5+2aZ9tXKauYXdKeiupcd6MPa9+Hik6/3Bj/3lnRH3GNFwzQhzv1YTvFFx1K+r6iulfei9W3ucrlJJ2szAvoXqAPa9eH1bWNPqzV+7CbpJWSjm6s/ivCPjxIX11Ad5ii/YrRj3X7e46XmagiuYCu4A2IO3SwojqYVxSdHl+o6ErGDpL+Kel1STMltY+X30dRDcxGSR/Hv++h6MrJRfHPUknX1bDNkxRd0flG5eUUXaj1RqV23FDN838Qb3ebotrWP1SaN0bSZPqwbn0oaT9JMxSd3l8i6Tv0Ya378Bvxel+V9LDiK9Hpx2r7scrlFO3E7o7nLZZUSh/Wug+rbBt9WKs+/IOkjypNX8D7sNZ9eE287YWS5ikqN2jwPiy2fsxaZqKKJAzzDXQAAAAIVpOpGQYAAAAaG2EYAAAAwSIMAwAAIFiEYQAAAASLMAwAAIBgEYYBAAAQLMIwAAAAgkUYBgAAQLD+P9ofL+W1268eAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 864x576 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "plt.figure(figsize=(12,8))\n",
        "plt.plot(for_plot[for_plot.sentiment=='POSITIVE']['tweet_created'], \n",
        "         for_plot[for_plot.sentiment=='POSITIVE']['sentiment_counts'], label='POSITIVE Tweets')\n",
        "plt.plot(for_plot[for_plot.sentiment=='NEGATIVE']['tweet_created'], \n",
        "         for_plot[for_plot.sentiment=='NEGATIVE']['sentiment_counts'], label='NEGATIVE Tweets')\n",
        "plt.legend()\n",
        "plt.title('Sentiment Analysis Over Time')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2ye8xsCsOkDg",
        "outputId": "cc710e40-4b5f-4f23-be29-223d4d5bef44"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "2015-02-23 10:58:58    3\n",
              "2015-02-24 09:54:34    3\n",
              "2015-02-20 16:10:03    2\n",
              "2015-02-23 21:18:02    2\n",
              "2015-02-21 13:16:08    2\n",
              "                      ..\n",
              "2015-02-19 07:34:25    1\n",
              "2015-02-19 07:34:40    1\n",
              "2015-02-19 07:35:08    1\n",
              "2015-02-19 07:35:44    1\n",
              "2015-02-22 11:58:51    1\n",
              "Name: tweet_created, Length: 8725, dtype: int64"
            ]
          },
          "execution_count": 78,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "tweets.tweet_created.value_counts()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "provenance": [],
      "gpuType": "A100"
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.12"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e73cf5f568e94ccb9ea5b8b183b5a649": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_93c3e699b455451e852fba9dbbfc8281",
              "IPY_MODEL_8eeb0b3b48af466eab828a6e8fef0e31",
              "IPY_MODEL_29bb6c3d6408417698e018c8cb9a9eb7"
            ],
            "layout": "IPY_MODEL_8850a4f6b3aa41c39099a6d02f79b96c"
          }
        },
        "93c3e699b455451e852fba9dbbfc8281": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c2f6e48e68b84133b9166d2dd5a863b4",
            "placeholder": "​",
            "style": "IPY_MODEL_482caf7196be4e9ea25a9778f21b5784",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "8eeb0b3b48af466eab828a6e8fef0e31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_13b80df8601b4c6d8b3f00335a37351a",
            "max": 629,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7503d7d83eaf4e5785fd6921f337e685",
            "value": 629
          }
        },
        "29bb6c3d6408417698e018c8cb9a9eb7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_879d3fc0b539414485c40deca6e84b26",
            "placeholder": "​",
            "style": "IPY_MODEL_613a24453e4e4c76b279b54ed6efc6d6",
            "value": " 629/629 [00:00&lt;00:00, 52.2kB/s]"
          }
        },
        "8850a4f6b3aa41c39099a6d02f79b96c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c2f6e48e68b84133b9166d2dd5a863b4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "482caf7196be4e9ea25a9778f21b5784": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "13b80df8601b4c6d8b3f00335a37351a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7503d7d83eaf4e5785fd6921f337e685": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "879d3fc0b539414485c40deca6e84b26": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "613a24453e4e4c76b279b54ed6efc6d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9a435a9366f042daaccf7c7750aa0df9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5e89c82af1cb4adcb7862070f5ca32a9",
              "IPY_MODEL_227e6fcd0fa747f5866a8a84a0f7d939",
              "IPY_MODEL_45ca48b1635341a6bd841e4086849a83"
            ],
            "layout": "IPY_MODEL_59df55a999b14f6c935435f59ee22e14"
          }
        },
        "5e89c82af1cb4adcb7862070f5ca32a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_66deeb09ca17424ebe4ff2ed13e7c4d4",
            "placeholder": "​",
            "style": "IPY_MODEL_3d7b5e411cb545a084808bfa45586c53",
            "value": "Downloading pytorch_model.bin: 100%"
          }
        },
        "227e6fcd0fa747f5866a8a84a0f7d939": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_08533a987f194c1988f116579c020702",
            "max": 267844284,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_944d14bd6a704fbd9958080472bcba04",
            "value": 267844284
          }
        },
        "45ca48b1635341a6bd841e4086849a83": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_46cf3bdda26841bcb7d31d8e6ff6e68c",
            "placeholder": "​",
            "style": "IPY_MODEL_30c0a143a4cf438599e73570fadfd061",
            "value": " 268M/268M [00:00&lt;00:00, 490MB/s]"
          }
        },
        "59df55a999b14f6c935435f59ee22e14": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "66deeb09ca17424ebe4ff2ed13e7c4d4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3d7b5e411cb545a084808bfa45586c53": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "08533a987f194c1988f116579c020702": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "944d14bd6a704fbd9958080472bcba04": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "46cf3bdda26841bcb7d31d8e6ff6e68c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "30c0a143a4cf438599e73570fadfd061": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "406a8e08aea94b6783b22f623eab0ad9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b81056338867489eaad84e735c398c00",
              "IPY_MODEL_5e5f78ead93448b6882e008077f24129",
              "IPY_MODEL_b9f29e8079824b1e96504dd9530c4aed"
            ],
            "layout": "IPY_MODEL_0df9b72076da4309a18c8ae086c11149"
          }
        },
        "b81056338867489eaad84e735c398c00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_62f21a55f61244c881ba7dc233f0c00e",
            "placeholder": "​",
            "style": "IPY_MODEL_609ebb90f4a94fa194a9afb1f35e33ab",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "5e5f78ead93448b6882e008077f24129": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d5cf96591576405684135e861905c643",
            "max": 48,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a13ff8427cf24bb392186fd3e910c390",
            "value": 48
          }
        },
        "b9f29e8079824b1e96504dd9530c4aed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_86080f75c4cf4d168aa62fcde2cf0843",
            "placeholder": "​",
            "style": "IPY_MODEL_938de54eebe14716910150f0b4dbb23f",
            "value": " 48.0/48.0 [00:00&lt;00:00, 4.52kB/s]"
          }
        },
        "0df9b72076da4309a18c8ae086c11149": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "62f21a55f61244c881ba7dc233f0c00e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "609ebb90f4a94fa194a9afb1f35e33ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d5cf96591576405684135e861905c643": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a13ff8427cf24bb392186fd3e910c390": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "86080f75c4cf4d168aa62fcde2cf0843": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "938de54eebe14716910150f0b4dbb23f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "103c42045dd24348b97b9e507716cc59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b2ed16ae612749a89f402c36d3269650",
              "IPY_MODEL_ed7fc4d5f54e4c92a50d06145638b810",
              "IPY_MODEL_0f5838fbf0dd4b2aa23bdbaf4f2e7caa"
            ],
            "layout": "IPY_MODEL_5fddb72c15da4b428a8da9141b456859"
          }
        },
        "b2ed16ae612749a89f402c36d3269650": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_65c28e9f3949455189d8bfddebfa1702",
            "placeholder": "​",
            "style": "IPY_MODEL_6f944832fbfa40a09777f149c65be8c8",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "ed7fc4d5f54e4c92a50d06145638b810": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4653a55265f44abfa8b2dc81d6fb7d54",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fa9b4f33572642cca0a3156d72ce6d1c",
            "value": 231508
          }
        },
        "0f5838fbf0dd4b2aa23bdbaf4f2e7caa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bc4af09af5d74edfa685c452f9612823",
            "placeholder": "​",
            "style": "IPY_MODEL_e421157b4b1c4e2bb25007cafe1d0efd",
            "value": " 232k/232k [00:00&lt;00:00, 10.2MB/s]"
          }
        },
        "5fddb72c15da4b428a8da9141b456859": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "65c28e9f3949455189d8bfddebfa1702": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6f944832fbfa40a09777f149c65be8c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4653a55265f44abfa8b2dc81d6fb7d54": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fa9b4f33572642cca0a3156d72ce6d1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bc4af09af5d74edfa685c452f9612823": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e421157b4b1c4e2bb25007cafe1d0efd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}