ITMTF_GoogleColab.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ITMTF_GoogleColab.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Jagaskak/CourseProject/blob/main/ITMTF_GoogleColab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GToajN2-aXJU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e86fba0a-bd70-41a4-b74f-34a4477fdd41"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive/')\n",
        "\n",
        "nytimes_fp = \"/content/drive/My Drive/Data/NYTimes/\"\n",
        "stockprices_fp = \"/content/drive/My Drive/Data/PriceHistory/\""
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SkFkgUC4YICe",
        "outputId": "608753a8-7e38-4219-b638-96e5e3a2c748"
      },
      "source": [
        "!pip install pyLDAvis\n",
        "\n",
        "import re\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from pprint import pprint\n",
        "import datetime\n",
        "from collections import defaultdict\n",
        "\n",
        "# NLTK\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "from nltk.corpus import stopwords\n",
        "from nltk import ngrams\n",
        "\n",
        "# Gensim\n",
        "import gensim\n",
        "from gensim import models\n",
        "import gensim.corpora as corpora\n",
        "from gensim.utils import simple_preprocess\n",
        "from gensim.models import CoherenceModel\n",
        "from gensim.models import Phrases\n",
        "\n",
        "# spacy for lemmatization\n",
        "import spacy\n",
        "\n",
        "# Plotting tools\n",
        "import pyLDAvis\n",
        "import pyLDAvis.gensim\n",
        "import matplotlib.pyplot as plt \n",
        "\n",
        "from scipy.stats import pearsonr\n",
        "\n",
        "import statsmodels\n",
        "from statsmodels.tsa.stattools import grangercausalitytests\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pyLDAvis\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a5/3a/af82e070a8a96e13217c8f362f9a73e82d61ac8fff3a2561946a97f96266/pyLDAvis-2.1.2.tar.gz (1.6MB)\n",
            "\u001b[K     |████████████████████████████████| 1.6MB 5.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: wheel>=0.23.0 in /usr/local/lib/python3.6/dist-packages (from pyLDAvis) (0.35.1)\n",
            "Requirement already satisfied: numpy>=1.9.2 in /usr/local/lib/python3.6/dist-packages (from pyLDAvis) (1.18.5)\n",
            "Requirement already satisfied: scipy>=0.18.0 in /usr/local/lib/python3.6/dist-packages (from pyLDAvis) (1.4.1)\n",
            "Requirement already satisfied: pandas>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from pyLDAvis) (1.1.4)\n",
            "Requirement already satisfied: joblib>=0.8.4 in /usr/local/lib/python3.6/dist-packages (from pyLDAvis) (0.17.0)\n",
            "Requirement already satisfied: jinja2>=2.7.2 in /usr/local/lib/python3.6/dist-packages (from pyLDAvis) (2.11.2)\n",
            "Requirement already satisfied: numexpr in /usr/local/lib/python3.6/dist-packages (from pyLDAvis) (2.7.1)\n",
            "Requirement already satisfied: pytest in /usr/local/lib/python3.6/dist-packages (from pyLDAvis) (3.6.4)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from pyLDAvis) (0.16.0)\n",
            "Collecting funcy\n",
            "  Downloading https://files.pythonhosted.org/packages/66/89/479de0afbbfb98d1c4b887936808764627300208bb771fcd823403645a36/funcy-1.15-py2.py3-none-any.whl\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.17.0->pyLDAvis) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.17.0->pyLDAvis) (2.8.1)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from jinja2>=2.7.2->pyLDAvis) (1.1.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from pytest->pyLDAvis) (50.3.2)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.6/dist-packages (from pytest->pyLDAvis) (20.3.0)\n",
            "Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.6/dist-packages (from pytest->pyLDAvis) (0.7.1)\n",
            "Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.6/dist-packages (from pytest->pyLDAvis) (1.4.0)\n",
            "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from pytest->pyLDAvis) (1.9.0)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from pytest->pyLDAvis) (1.15.0)\n",
            "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from pytest->pyLDAvis) (8.6.0)\n",
            "Building wheels for collected packages: pyLDAvis\n",
            "  Building wheel for pyLDAvis (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyLDAvis: filename=pyLDAvis-2.1.2-py2.py3-none-any.whl size=97712 sha256=d70944d0ebd0a3375e2be23931d6a8c7ba78da8294aee8c5605c6b6acc34a802\n",
            "  Stored in directory: /root/.cache/pip/wheels/98/71/24/513a99e58bb6b8465bae4d2d5e9dba8f0bef8179e3051ac414\n",
            "Successfully built pyLDAvis\n",
            "Installing collected packages: funcy, pyLDAvis\n",
            "Successfully installed funcy-1.15 pyLDAvis-2.1.2\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l95VIW6Ire5B"
      },
      "source": [
        "'''\n",
        "This cell contains the processing of NYTimes data and IEM stock market data. You should only run this once.\n",
        "'''\n",
        "\n",
        "stop_words = stopwords.words('english')\n",
        "stop_words.extend(['mr', 'ms', 'mrs', 'said'])\n",
        "\n",
        "# Tokenize and remove stop words from content\n",
        "def tokenize(content, lemmatize=False):\n",
        "    words = gensim.utils.simple_preprocess(content, deacc=True)  # tokenizes\n",
        "    return words\n",
        "\n",
        "def remove_stopwords(content):\n",
        "    words = []\n",
        "    for word in content:\n",
        "        if word in stop_words:\n",
        "            continue\n",
        "        words.append(word)\n",
        "    return words\n",
        "\n",
        "\n",
        "'''\n",
        "Retrieve Data from files\n",
        "'''\n",
        "\n",
        "# New York Times Data\n",
        "rows = []\n",
        "dates = []\n",
        "articles = []\n",
        "for month in range(5, 11):\n",
        "    with open(nytimes_fp + str(month) + \".txt\") as f:\n",
        "        for i, line in enumerate(f):\n",
        "            date, article = line.split(\",\", 1)\n",
        "            timestamp = datetime.datetime.strptime(date, \"%Y-%m-%dT%H:%M:%S%z\").date()\n",
        "            tokenized = tokenize(article)\n",
        "            destopped = remove_stopwords(tokenized)\n",
        "\n",
        "            articles.append(destopped)\n",
        "            dates.append(timestamp)\n",
        "            rows.append([timestamp, destopped])\n",
        "\n",
        "nytimes = pd.DataFrame(rows, columns=[\"Date\", \"Content\"]) \n",
        "unique_dates = sorted(list(set(nytimes[\"Date\"])))\n",
        "# print (unique_dates)\n",
        "\n",
        "# Time Series Data\n",
        "ts_months = [\"May\", \"Jun\", \"Jul\", \"Aug\", \"Sep\", \"Oct\"]\n",
        "cols = ['Date', 'LastPrice']\n",
        "stock_prices = pd.DataFrame()\n",
        "for month in ts_months:\n",
        "    ts_df = pd.read_csv(stockprices_fp + month + \".txt\", delim_whitespace=True)\n",
        "    ts_df['Date'] =  ts_df['Date'].apply(lambda x: datetime.datetime.strptime(x, \"%m/%d/%y\").date())\n",
        "    \n",
        "    Gore = ts_df.loc[ts_df['Contract'] == 'Dem'][['Date', 'LastPrice']].fillna(0).reset_index()\n",
        "    Bush = ts_df.loc[ts_df['Contract'] == 'Rep'][['Date', 'LastPrice']].fillna(0).reset_index()\n",
        "\n",
        "    # Gore/(Gore + Bush)\n",
        "    relation = list(zip(Gore['Date'], (Gore['LastPrice']/(Gore['LastPrice'] + Bush['LastPrice'])).fillna(0.001)))\n",
        "    stock_prices = stock_prices.append(relation, ignore_index=True)\n",
        "\n",
        "stock_prices.columns = cols\n",
        "\n",
        "\n",
        "'''\n",
        "BOW - corpus, date x word cnts\n",
        "'''\n",
        "# Create Dictionary\n",
        "id2word = corpora.Dictionary(articles)\n",
        "\n",
        "# Filtering: \n",
        "# Keep words that appear in at least 1% of docs --> approximately 50 topics\n",
        "# Don't keep words that appear in more that 70% of docs in corpus\n",
        "# id2word.filter_extremes(no_below=50, no_above=0.7)\n",
        "\n",
        "# Term Document Frequency\n",
        "corpus = [id2word.doc2bow(text) for text in articles]\n",
        "\n",
        "# TF-IDF seems to give better coherence (but it wasn't in the paper...)\n",
        "tfidf = models.TfidfModel(corpus)\n",
        "tfidf_corpus = tfidf[corpus]\n",
        "\n",
        "# Human readable format of corpus (term-frequency)\n",
        "# [[(id2word[id], freq) for id, freq in cp] for cp in tfidf_corpus[:1]][0][:5]\n",
        "\n",
        "# bow by date\n",
        "date_term_cnts = defaultdict(lambda: [])\n",
        "\n",
        "for index, row in nytimes.iterrows():\n",
        "    date = row[\"Date\"]\n",
        "    content = row[\"Content\"]\n",
        "    \n",
        "    date_term_cnts[date] += content\n",
        "    \n",
        "date_term_cnts = list(date_term_cnts.items())\n",
        "date_term_cnts = [(date, {id2word[id]: freq for id, freq in id2word.doc2bow(text)}) for date, text in date_term_cnts]\n",
        "date_term_cnts = sorted(date_term_cnts, key=lambda x: x[0])\n",
        "date_term_cnts = pd.DataFrame([date_term_cnts[i][1] for i in range(len(date_term_cnts))], index=[date_term_cnts[i][0] for i in range(len(date_term_cnts))]).fillna(0.0)\n"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iuJ711r8YLYc"
      },
      "source": [
        "'''\n",
        "This cell contains function definitions.\n",
        "'''\n",
        "\n",
        "def rel_purity(currPurity, prevPurity):\n",
        "    return abs(currPurity - prevPurity)/prevPurity\n",
        "\n",
        "\n",
        "# Select the model and print the topics\n",
        "def get_topics(lda_model, num_topics=-1, num_words=100, prob_thresh=0.8):\n",
        "    topics = []\n",
        "    for topic, topic_words in lda_model.print_topics(num_topics=num_topics, num_words=num_words):\n",
        "        words = topic_words.split(\" + \")\n",
        "        all_words = []\n",
        "        all_prob = 0\n",
        "        for elem in words:\n",
        "            prob, word = elem.split(\"*\")\n",
        "            all_prob += float(prob)\n",
        "            all_words.append(word.split('\"')[1])\n",
        "\n",
        "            if all_prob >= prob_thresh:\n",
        "                break\n",
        "        topics.append((topic, all_words))\n",
        "    \n",
        "    return topics\n",
        "\n",
        "\n",
        "'''\n",
        "Significant portion of grangers_causality_matrix function was taken from stackoverflow post:\n",
        "https://stackoverflow.com/questions/58005681/is-it-possible-to-run-a-vector-autoregression-analysis-on-a-large-gdp-data-with\n",
        "'''\n",
        "def grangers_causality_matrix(data, variables, maxlag=5, test='ssr_ftest', verbose=False):\n",
        "    dataset = pd.DataFrame(np.zeros((len(variables), len(variables))), columns=variables, index=variables)\n",
        "    lags    = pd.DataFrame(np.zeros((len(variables), len(variables))), columns=variables, index=variables)\n",
        "    \n",
        "    for c in dataset.columns:\n",
        "        for r in dataset.index:            \n",
        "            test_result = grangercausalitytests(data[[r,c]], maxlag=maxlag, verbose=False)\n",
        "            p_values = [round(test_result[i+1][0][test][1], 4) for i in range(maxlag)]\n",
        "            \n",
        "            if verbose: \n",
        "                print(f'Y = {r}, X = {c}, P Values = {p_values}')\n",
        "\n",
        "            # smaller p-val corresponds to higher f-val\n",
        "            min_p_value_i = np.argmin(p_values)\n",
        "            min_p_value = p_values[min_p_value_i]\n",
        "            dataset.loc[r, c] = min_p_value\n",
        "            \n",
        "            lags.loc[r, c] = min_p_value_i\n",
        "   \n",
        "    return dataset, lags\n",
        "\n",
        "def get_causal_vars(data, significance=0.95, getLags=False, getCausalSig=False, verbose=False):\n",
        "    cols = data.columns[:-1]\n",
        "    causal_vars = []\n",
        "    causal_lags = []\n",
        "    \n",
        "    for col in cols:\n",
        "        try:\n",
        "            gc, lags = grangers_causality_matrix(data[[col, 'LastPrice']], \n",
        "                                             variables=[col, 'LastPrice'], \n",
        "                                             verbose=False)\n",
        "        except:\n",
        "            raise Exception(data[[col, 'LastPrice']])\n",
        "        \n",
        "        gc = 1 - gc\n",
        "        \n",
        "        col_causes = gc.loc['LastPrice', col] >= significance\n",
        "        col_causedBy = gc.loc[col, 'LastPrice'] >= significance\n",
        "        if col_causes or col_causedBy:\n",
        "            if getCausalSig:\n",
        "                causal_vars.append((col, max(gc.loc['LastPrice', col], gc.loc[col, 'LastPrice'])))\n",
        "            else:\n",
        "                causal_vars.append(col)\n",
        "            \n",
        "            if getLags:\n",
        "                # if granger causality for topic causing ts and ts causing topic both significant, choose higher sig\n",
        "                if col_causes and col_causedBy:\n",
        "                    if gc.loc['LastPrice', col] >= gc.loc[col, 'LastPrice']:\n",
        "                        causal_lags.append(lags.loc['LastPrice', col])\n",
        "                    else:\n",
        "                        causal_lags.append(lags.loc[col, 'LastPrice'] * -1)\n",
        "                elif col_causes:\n",
        "                    causal_lags.append(lags.loc['LastPrice', col])\n",
        "                else:\n",
        "                    causal_lags.append(lags.loc[col, 'LastPrice'] * -1)\n",
        "    if getLags:\n",
        "        return causal_vars, causal_lags\n",
        "    \n",
        "    return causal_vars\n",
        "                \n",
        "def get_word_stream(nytimes, topics, causal_topics):\n",
        "    ct_ws = []\n",
        "    for ct in causal_topics:\n",
        "        causal_vocab = list(set(topics[ct][1]))\n",
        "        date_terms = pd.DataFrame(np.zeros((len(unique_dates), len(causal_vocab))), index=unique_dates, columns=causal_vocab)\n",
        "        \n",
        "        for word in causal_vocab:\n",
        "            date_terms[word] = date_term_cnts[word]\n",
        "            \n",
        "        ct_ws.append((ct, date_terms))\n",
        "    \n",
        "    return ct_ws\n",
        "\n",
        "def get_impact_words(topic_wordstream, significance=0.95, verbose=False):\n",
        "    topic_impact_words = []\n",
        "    \n",
        "    for topic, ws in topic_wordstream:\n",
        "        ws_prices = ws.join(stock_prices.set_index('Date')).dropna()        \n",
        "        ws_gc = get_causal_vars(ws_prices, significance=significance, getCausalSig=True, verbose=verbose)\n",
        "        \n",
        "        pos = []\n",
        "        neg = []\n",
        "        for word, sig in ws_gc:                \n",
        "            corr = pearsonr(ws_prices[word], stock_prices['LastPrice'])[0]\n",
        "            if corr >= 0:\n",
        "                pos.append((word, sig))\n",
        "            else:\n",
        "                neg.append((word, sig))\n",
        "                \n",
        "        topic_impact_words.append((topic, pos, neg))\n",
        "    \n",
        "    return topic_impact_words\n",
        "        \n",
        "def construct_prior(impact_words, curr_k, const_k_increase=0, sig=0.95, alter_k=True):\n",
        "    # find number of topics that we are splitting\n",
        "    if alter_k:\n",
        "        new_k = curr_k + len(impact_words) + const_k_increase\n",
        "    else:\n",
        "        new_k = curr_k\n",
        "    word_priors = np.zeros((new_k, date_term_cnts.shape[1])) + (1/len(id2word))\n",
        "\n",
        "    i = 0\n",
        "    for topic_num, pos, neg in impact_words:\n",
        "        pos_denom = sum([granger-sig for word, granger in pos])\n",
        "        neg_denom = sum([granger-sig for word, granger in neg])\n",
        "        \n",
        "        if len(pos) < 0.1 * len(neg):\n",
        "            # num neg words >> num pos\n",
        "            for word, granger in pos:              \n",
        "                word_priors[i, id2word.token2id[word]] = 0\n",
        "            for word, granger in neg:\n",
        "                word_priors[i, id2word.token2id[word]] = (granger-sig)/neg_denom \n",
        "            i += 1\n",
        "            continue\n",
        "            \n",
        "        elif len(neg) < 0.1 * len(pos):\n",
        "            # num pos words >> num neg\n",
        "            for word, granger in pos:              \n",
        "                word_priors[i, id2word.token2id[word]] = (granger-sig)/pos_denom \n",
        "            for word, granger in neg:\n",
        "                word_priors[i, id2word.token2id[word]] = 0\n",
        "            i += 1\n",
        "            continue\n",
        "            \n",
        "        for word, granger in pos:              \n",
        "            word_priors[i, id2word.token2id[word]] = (granger-sig)/pos_denom \n",
        "        \n",
        "        for word, granger in neg:\n",
        "            word_priors[i + 1, id2word.token2id[word]] = (granger-sig)/neg_denom \n",
        "        \n",
        "        i += 2\n",
        "    \n",
        "    return word_priors\n",
        "            \n",
        "def calculate_purity(pWords, nWords):\n",
        "    n = float(len(pWords) + len(nWords))\n",
        "    if n == 0:\n",
        "        return 0\n",
        "    pProb = len(pWords)/n\n",
        "    nProb = len(nWords)/n\n",
        "    \n",
        "    pProb = pProb if pProb else 1\n",
        "    nProb = nProb if nProb else 1\n",
        "    \n",
        "    entropy = pProb * np.log(pProb) + nProb * np.log(nProb)\n",
        "    purity = 100 + 100 * entropy\n",
        "    return purity\n",
        "\n",
        "def calculate_conf(impact_words):\n",
        "    all_conf = 0.0\n",
        "    num_words = 0\n",
        "    for topic, pos, neg in impact_words:\n",
        "        num_words += len(pos) + len(neg)\n",
        "        for words, gc in pos:\n",
        "            all_conf += gc\n",
        "\n",
        "        for words, gc in neg:\n",
        "            all_conf += gc\n",
        "\n",
        "    return all_conf/num_words*100 if num_words else 0\n",
        "\n",
        "\n",
        "def show_plot(x, yData, xaxislabel, yaxislabel, labels, xticks=None, yaxisrange=None, title=None, legend_title=None, marker=\".\", saveAs=None):\n",
        "    if len(yData) != len(labels):\n",
        "        raise ValueError(\"Number of labels should equal number of lines you want to plot\")\n",
        "    \n",
        "    plt.xlabel(xaxislabel)\n",
        "    plt.ylabel(yaxislabel)\n",
        "    plt.grid()\n",
        "        \n",
        "    for i in range(len(yData)):\n",
        "        plt.plot(x, yData[i], label=labels[i], marker=marker)\n",
        "    plt.legend(loc=2, bbox_to_anchor=(1.05, 1), title=legend_title)\n",
        "    \n",
        "    if yaxisrange:\n",
        "        plt.ylim(yaxisrange)\n",
        "    if xticks:\n",
        "        plt.xticks(xticks)\n",
        "    if title:\n",
        "        plt.title(title)\n",
        "    \n",
        "    if saveAs:\n",
        "        plt.savefig(saveAs, bbox_inches='tight')\n",
        "    plt.show()"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h0fJqMhVZKt1",
        "outputId": "094d4fda-83c3-4948-de3f-091a5bdecf67"
      },
      "source": [
        "'''\n",
        "This function will run topic modeling with time series feedback \n",
        "\n",
        "Parameters:\n",
        "paramCtrl  -- str (either 'k' or 'decay'), tells function to test various k or decay values\n",
        "params     -- list of integers, each val in list is either a k or decay values to test.\n",
        "              For k values, the last k value will be used to test a variable number of topics starting at k topics.\n",
        "              In every iteration of ITMTF, k will increase by the number of causal topics found + some constant\n",
        "decay      -- (0.5, 1] decay parameter to use when running lda (effective only when paramCtrl='k')\n",
        "k          -- Interger representing number of topics to use when running lda (effective only when paramCtrl='decay')\n",
        "iterations       -- Integer number of iterations to run ITMTF\n",
        "const_k_increase -- Integer number of topics to increase (in addition to number of causal topics found) \n",
        "                    each iteration when running varying number of topics (effective only when paramCtrl='k')\n",
        "                    Default is 0\n",
        "verbose          -- True/False Print out purity and confidence every iteration\n",
        "\n",
        "Returns:\n",
        "k_lda_model    -- The final lda_model  \n",
        "k_avg_purities -- Average purity of all causal topics in each iteration\n",
        "k_avg_conf     -- Average confidence of all causal topics in each iteration\n",
        "'''\n",
        "def ITMTF(paramCtrl, params, decay=1, k=30, const_k_increase=5, iterations=5, verbose=True):\n",
        "    thirdLDAModel = None\n",
        "    \n",
        "    prevPurity = 100\n",
        "    purity = 0\n",
        "    alpha = \"auto\"\n",
        "    eta = \"auto\"\n",
        "    alter_k = False\n",
        "\n",
        "    all_avg_purities = []\n",
        "    all_avg_conf = []\n",
        "\n",
        "    for val in params:\n",
        "        if paramCtrl == \"k\":\n",
        "            k = val\n",
        "        elif paramCtrl == \"decay\":\n",
        "            decay = val\n",
        "        else:\n",
        "            raise ValueError(\"Invalid argument. Please read docs to understand use case\")\n",
        "        \n",
        "        if verbose:\n",
        "            print (\"\\n\\nk: \", k)\n",
        "            print (\"decay: \", decay)\n",
        "\n",
        "        avg_purities = []\n",
        "        avg_confidences = []\n",
        "\n",
        "        num_iter = 0\n",
        "        while num_iter < iterations:\n",
        "\n",
        "            if verbose and alter_k == \"k\":\n",
        "                print (\"\\nNumber topics: \", k)          \n",
        "\n",
        "            lda_model = gensim.models.ldamodel.LdaModel(\n",
        "                                    corpus=tfidf_corpus,\n",
        "                                    id2word=id2word,\n",
        "                                    num_topics=k, \n",
        "                                    alpha=alpha,\n",
        "                                    eta=eta,\n",
        "                                    decay=decay)\n",
        "\n",
        "            topics = get_topics(lda_model, prob_thresh=0.3, num_words=50)\n",
        "            document_topics = lda_model.get_document_topics(corpus)\n",
        "            date_doc_topics = list(zip(nytimes[\"Date\"], lda_model.get_document_topics(corpus)))\n",
        "\n",
        "            # for any given day, you look at all the diff topics and identify the prob of that topic\n",
        "            date_topic_prob = np.zeros((len(unique_dates), k))\n",
        "            for date, article in date_doc_topics:\n",
        "                i = unique_dates.index(date)\n",
        "                for topic, prob in article:\n",
        "                    date_topic_prob[i][topic] += prob \n",
        "\n",
        "            date_topic = pd.DataFrame(date_topic_prob, index=unique_dates)\n",
        "            date_topic[\"Date\"] = unique_dates\n",
        "\n",
        "            date_topic_prices = date_topic.set_index('Date').join(stock_prices.set_index('Date')).dropna()\n",
        "            causal_topics = get_causal_vars(date_topic_prices)\n",
        "            ct_ws = get_word_stream(nytimes, topics, causal_topics)\n",
        "\n",
        "            impact_words = get_impact_words(ct_ws)\n",
        "\n",
        "            # Calculate Purity\n",
        "            purities = [calculate_purity(topic[1], topic[2]) for topic in impact_words]\n",
        "            if len(purities) == 0:\n",
        "                # Try again, we got no causal topics\n",
        "                continue\n",
        "\n",
        "            avg_purity = sum(purities)/len(purities)\n",
        "            prevPurity = purity\n",
        "            purity = avg_purity\n",
        "\n",
        "            avg_purities.append(purity)\n",
        "\n",
        "            # Calculate Confidence\n",
        "            avg_conf = calculate_conf(impact_words)\n",
        "            avg_confidences.append(avg_conf)\n",
        "            \n",
        "            if verbose:\n",
        "                print (\"Purity: \", purity)  \n",
        "                print (\"Avg. Conf: \", avg_conf)\n",
        "\n",
        "            # Prior for next iteration\n",
        "            eta = construct_prior(impact_words, k, const_k_increase=const_k_increase, alter_k=alter_k)\n",
        "\n",
        "            # adjust num topics\n",
        "            if alter_k:\n",
        "                k += len(impact_words) + const_k_increase\n",
        "\n",
        "    #         if keep3rd and (i + 1) == 3:\n",
        "    #             thirdLDAModel = lda_model\n",
        "            num_iter += 1\n",
        "\n",
        "        all_avg_purities.append(avg_purities)\n",
        "        all_avg_conf.append(avg_confidences)\n",
        "\n",
        "        # reset params\n",
        "        eta = \"auto\"\n",
        "\n",
        "        if paramCtrl == \"k\":\n",
        "            alter_k = (k == params[-2])\n",
        "\n",
        "    return lda_model, all_avg_purities, all_avg_conf\n",
        "\n",
        "k_lda_model, k_avg_purities, k_avg_conf = ITMTF(\"k\", params=[10, 20, 30, 40, 10], const_k_increase=3)\n",
        "# mu_lda_model, mu_avg_purities, mu_avg_conf = ITMTF(\"decay\", params=[0.51, 0.6, 0.7, 0.8, 0.9, 1])"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "k:  10\n",
            "decay:  1\n",
            "Purity:  76.22590918988769\n",
            "Avg. Conf:  97.12599999999998\n",
            "Purity:  45.830909574139845\n",
            "Avg. Conf:  97.51153846153848\n",
            "Purity:  43.35827545160637\n",
            "Avg. Conf:  96.61866666666667\n",
            "Purity:  52.451818379775375\n",
            "Avg. Conf:  97.03545454545453\n",
            "Purity:  77.47193955668477\n",
            "Avg. Conf:  96.95499999999998\n",
            "\n",
            "\n",
            "k:  20\n",
            "decay:  1\n",
            "Purity:  31.709189529952823\n",
            "Avg. Conf:  97.75999999999999\n",
            "Purity:  54.581998589112345\n",
            "Avg. Conf:  97.43473684210527\n",
            "Purity:  40.65924807596458\n",
            "Avg. Conf:  97.86999999999999\n",
            "Purity:  61.84409152221347\n",
            "Avg. Conf:  97.3885714285714\n",
            "Purity:  62.16007013052867\n",
            "Avg. Conf:  97.61869565217391\n",
            "\n",
            "\n",
            "k:  30\n",
            "decay:  1\n",
            "Purity:  62.80194994145747\n",
            "Avg. Conf:  98.09214285714285\n",
            "Purity:  47.819697347843\n",
            "Avg. Conf:  97.87491525423727\n",
            "Purity:  43.30216381409163\n",
            "Avg. Conf:  97.77888888888886\n",
            "Purity:  56.22351014010727\n",
            "Avg. Conf:  98.01142857142858\n",
            "Purity:  43.9130564413375\n",
            "Avg. Conf:  97.9355172413793\n",
            "\n",
            "\n",
            "k:  40\n",
            "decay:  1\n",
            "Purity:  56.62183857592694\n",
            "Avg. Conf:  97.80181818181819\n",
            "Purity:  55.69359188602986\n",
            "Avg. Conf:  97.56420454545459\n",
            "Purity:  45.162033596607145\n",
            "Avg. Conf:  97.88421052631581\n",
            "Purity:  60.76409437829668\n",
            "Avg. Conf:  97.84419354838711\n",
            "Purity:  41.62109160649805\n",
            "Avg. Conf:  97.6522680412371\n",
            "\n",
            "\n",
            "k:  10\n",
            "decay:  1\n",
            "Purity:  46.86312159215019\n",
            "Avg. Conf:  97.59230769230768\n",
            "Purity:  65.93854123825706\n",
            "Avg. Conf:  97.49944444444445\n",
            "Purity:  61.63490167342927\n",
            "Avg. Conf:  97.58333333333333\n",
            "Purity:  46.8060886268385\n",
            "Avg. Conf:  97.81138888888887\n",
            "Purity:  51.88749004332235\n",
            "Avg. Conf:  98.044\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vRBH0GV0W19W"
      },
      "source": [
        "'''\n",
        "Plotting purity and confidence for different number of topics\n",
        "'''\n",
        "\n",
        "k_labels = [\"10\", \"20\", \"30\", \"40\", \"Varying Number of Topics\"]\n",
        "show_plot(range(1, 6), k_avg_purities, \n",
        "          \"Iteration\", \"Average Purity\",     \n",
        "          k_labels, \n",
        "          xticks=range(1, 6), yaxisrange=[0, 120], \n",
        "          title=\"Average Confidence for Different Number of Topics\",\n",
        "          legend_title=\"Number of Topics\",\n",
        "          saveAs=\"purity_k.png\")\n",
        "show_plot(range(1, 6), k_avg_conf,     \n",
        "          \"Iteration\", \"Average Confidence\", \n",
        "          k_labels, \n",
        "          xticks=range(1, 6), yaxisrange=[95.5, 100], \n",
        "          title=\"Average Confidence for Different Number of Topics\",\n",
        "          legend_title=\"Number of Topics\",\n",
        "          saveAs=\"confidence_k.png\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D-5wOMq8ri_W"
      },
      "source": [
        "'''\n",
        "Plotting purity and confidence for different values of decay\n",
        "'''\n",
        "\n",
        "mu_labels = [\"0.51\", \"0.6\", \"0.7\", \"0.8\", \"0.9\", \"1\"]\n",
        "show_plot(range(1, 6), mu_avg_purities, \n",
        "          \"Iteration\", \"Average Purity\", \n",
        "          mu_labels, \n",
        "          xticks=range(1, 6), yaxisrange=[0, 120], \n",
        "          title=\"Average Purity for Different Decay Values\", \n",
        "          legend_title=\"Decay Values\", \n",
        "          saveAs=\"purity_mu.png\")\n",
        "show_plot(range(1, 6), mu_avg_conf, \n",
        "          \"Iteration\", \"Average Confidence\", \n",
        "          mu_labels, \n",
        "          xticks=range(1, 6), yaxisrange=[95.5, 100], \n",
        "          title=\"Average Confidence for Different Decay Values\", \n",
        "          legend_title=\"Decay Values\", \n",
        "          saveAs=\"confidence_mu.png\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BvKyC24KbFtV"
      },
      "source": [
        "# Visualize the topic model results lda model (w/ varying number of topics) using pyLDAvis\n",
        "pyLDAvis.enable_notebook()\n",
        "vis = pyLDAvis.gensim.prepare(k_lda_model, corpus, id2word)\n",
        "pyLDAvis.save_html(vis, 'LDAvis_k.html')\n",
        "vis"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fcq6hyYcJdsF"
      },
      "source": [
        "# Visualize the topic model results lda model (w/ mu=1 and k=30) using pyLDAvis\n",
        "pyLDAvis.enable_notebook()\n",
        "vis = pyLDAvis.gensim.prepare(mu_lda_model, corpus, id2word)\n",
        "pyLDAvis.save_html(vis, 'LDAvis_mu.html')\n",
        "vis"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}