ex9_intro_to_hlt_2023.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "f92e1aa1078243c4bdb51751bc73c77d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_81e2f35558a8424abd221b30ca2d7c54",
              "IPY_MODEL_41fb7d881e58466eb06113d453957dce",
              "IPY_MODEL_d45fbe39c51148ab98dec20032089984"
            ],
            "layout": "IPY_MODEL_1aa4b45250f74ff690dabf99f77edf44"
          }
        },
        "81e2f35558a8424abd221b30ca2d7c54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a6cc892d2c27436e89cece3c599d01e5",
            "placeholder": "​",
            "style": "IPY_MODEL_cb641944d73a49f79f4fb2d7e42ba869",
            "value": "100%"
          }
        },
        "41fb7d881e58466eb06113d453957dce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_136c121282dc450284088e8650d5ab88",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_eb4cb31d4ae44db78908d08c76132be7",
            "value": 3
          }
        },
        "d45fbe39c51148ab98dec20032089984": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e6171ba83c1543f990c57a2ffea2fafb",
            "placeholder": "​",
            "style": "IPY_MODEL_66c75d7b6e124bcebabb58fc24feef93",
            "value": " 3/3 [00:00&lt;00:00, 30.89it/s]"
          }
        },
        "1aa4b45250f74ff690dabf99f77edf44": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a6cc892d2c27436e89cece3c599d01e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cb641944d73a49f79f4fb2d7e42ba869": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "136c121282dc450284088e8650d5ab88": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eb4cb31d4ae44db78908d08c76132be7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e6171ba83c1543f990c57a2ffea2fafb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "66c75d7b6e124bcebabb58fc24feef93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TurkuNLP/intro-to-nlp/blob/master/ex9_intro_to_hlt_2023.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "In this exercise, you'll try to generate text with an n-gram model.\n",
        "\n",
        "To keep things at least a little bit efficient and be able to generate with up to 4-grams in google colab memory, let us divide the task as follows:\n",
        "\n",
        "1. Generate n-grams from a corpus of text, e.g. the IMDB dataset\n",
        "2. Count the n-grams, i.e. get a Counter with all unique n-grams and their counts in the text\n",
        "3. From the Counter, we then build a dictionary, where for each n-1 -gram, we have a list of all words which can continue it. For example for the 4-gram \"I have a dog\", this dictionary will have an entry where the key is *(I,have,a)* and the value is a list which contains *dog* together with all other words *X* that were seen in a 4-gram *I have a X*. Put in other words, the dictionary encodes all ways in which a n-1 gram can be continued that we saw in the data.\n",
        "\n",
        "With these data structures, the generation can proceed quite easily. Say, we have a 4-gram model.\n",
        "\n",
        "* Given a prior context $w_1w_2w_3$\n",
        "* Look up the list of possible words $w_4$ in the dictionary from step 3, then for each word, the count of $w_1w_2w_3w_4$ can be looked up in the Counter from step 2\n",
        "* The counts, once normalized to sum up to 1, form a distribution over words that can continue $w_1w_2w_3$ and we can sample the next word from this distribution.\n",
        "\n",
        "Other remarks:\n",
        "\n",
        "* We want to pad all texts with <bos> (beginning of sequence) and <eos> (end of sequence). The <bos> we want to have there n-1 times, so we can use it as the initial prompt and let the model learn how the sequences start. The <eos> allows us to stop generating, and prevents a crash on unknown n-grams at the very end of a sequence. (if an n-gram $w_1w_2w_3w_4$ was seen only once at the end of a \"training\" sequence, then an attempt to continue it during generation, would lead to a crash, since we have no known n-gram to continue the sequence $w_2w_3w_4$ with our simple, unsmoothed model :)\n"
      ],
      "metadata": {
        "id": "nF1-YWS82oKK"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Task A: Generate n-grams\n",
        "\n",
        "* Write a generator function `generate_ngrams(dset,n)` (using `yield` rather than `return`) which yields n-grams as tuples $(w_1,...,w_n)$ from all sections of the IMDB dataset\n",
        "* a vectorizer from `sklearn` can be used as a trivial tokenizer\n",
        "* `more-itertools` is a nifty library to achieve the n-gram generation\n",
        "* remember to pad with n-1 `<bos>` symbols at the beginning, and one `<eos>` symbol at the end\n",
        "\n",
        "You can give this a shot, or simply use the code for task A below to move on to the juicier parts of the exercise. So, warning, spoiler below."
      ],
      "metadata": {
        "id": "qD9E2rDD7oyE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip3 install datasets more-itertools"
      ],
      "metadata": {
        "id": "a4efOx1BpIN9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WklQqyCFo_aq"
      },
      "outputs": [],
      "source": [
        "import datasets\n",
        "import sklearn.feature_extraction"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dset=datasets.load_dataset(\"imdb\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86,
          "referenced_widgets": [
            "f92e1aa1078243c4bdb51751bc73c77d",
            "81e2f35558a8424abd221b30ca2d7c54",
            "41fb7d881e58466eb06113d453957dce",
            "d45fbe39c51148ab98dec20032089984",
            "1aa4b45250f74ff690dabf99f77edf44",
            "a6cc892d2c27436e89cece3c599d01e5",
            "cb641944d73a49f79f4fb2d7e42ba869",
            "136c121282dc450284088e8650d5ab88",
            "eb4cb31d4ae44db78908d08c76132be7",
            "e6171ba83c1543f990c57a2ffea2fafb",
            "66c75d7b6e124bcebabb58fc24feef93"
          ]
        },
        "id": "BAiGkrtqpYo1",
        "outputId": "92f4d34c-455c-4557-cebe-74f32de5884b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.builder:Found cached dataset imdb (/root/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f92e1aa1078243c4bdb51751bc73c77d"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Few remarks here:\n",
        "# 1. we don't need the vectorizer per se, we just want its analyzer function, which basically tokenizes the text for us, and somewhat unfortunately drops punctuation\n",
        "# 2. the default token pattern in sklearn drops 1-letter words (like \"I\" and \"a\") so I modify it a bit\n",
        "cvectorizer=sklearn.feature_extraction.text.CountVectorizer(lowercase=False,stop_words=None,token_pattern=r\"(?u)\\b\\w+\\b\" )\n",
        "analyzer=cvectorizer.build_analyzer()\n",
        "analyzer(\"I have a dog at home, it likes to shred newspapers.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HeKF44p4pjyE",
        "outputId": "fa422e01-35a4-4ee8-96e8-49dfe12a2eb1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['I',\n",
              " 'have',\n",
              " 'a',\n",
              " 'dog',\n",
              " 'at',\n",
              " 'home',\n",
              " 'it',\n",
              " 'likes',\n",
              " 'to',\n",
              " 'shred',\n",
              " 'newspapers']"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Now we tokenize the IMDB dataset the usual way\n",
        "def tokenize(ex):\n",
        "    return {\"tokenized\":analyzer(ex[\"text\"])}\n",
        "\n",
        "dset=dset.map(tokenize,num_proc=4)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eCdn2w-drcVN",
        "outputId": "594cbc0f-eaf1-4552-aefa-bdb4ef5c00c6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0/cache-d09f1293d5219067_*_of_00004.arrow\n",
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0/cache-575cdb577c85a33d_*_of_00004.arrow\n",
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0/cache-9cafbb586a726924_*_of_00004.arrow\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from collections import Counter\n",
        "from more_itertools import sliding_window #more-itertools is an awesome library!\n",
        "import tqdm\n",
        "\n",
        "def generate_ngrams(dset,n):\n",
        "    for ex in tqdm.tqdm(dset):\n",
        "        tokens=[\"<bos>\"]*(n-1)+ex[\"tokenized\"]+[\"<eos>\"]\n",
        "        for ngram in sliding_window(tokens,n):\n",
        "            yield ngram\n",
        "\n"
      ],
      "metadata": {
        "id": "OPXGteLQtOXD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Task B\n",
        "\n",
        "* Now we can combine the different sections of the IMDB dataset and count our n-grams\n",
        "* That is relatively easy since we write generate_ngrams() as a generator and collections.Counter can count it directly"
      ],
      "metadata": {
        "id": "LOoapsbtAcyM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Here we can concatenate all the individual datasets (train,test,unlabeled) in IMDB\n",
        "# the \"master\" dataset is a dictionary of these, so dset.values() has the datasets of the individual sections (train,test,unlabeled)\n",
        "#\n",
        "# Please make sure you visit the datasets documentation page and understand what this does!\n",
        "combined_dataset=datasets.concatenate_datasets(list(dset.values()))\n"
      ],
      "metadata": {
        "id": "UiuqC8yl_lqo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ngrams=### THIS YOU NEED TO FILL IN :)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B4qumygtAYZH",
        "outputId": "ac56d06e-e3b3-45dc-ae9c-87b6db3a5b52"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 100000/100000 [01:14<00:00, 1339.74it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Task C\n",
        "\n",
        "* Now we have ngrams, which is a simple counter\n",
        "* So we can iterate over its keys, and build the dictionary of possible continuations"
      ],
      "metadata": {
        "id": "tw1xMSkBA3HP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "cont_dict={} #key is n-1 gram, as a tuple, like (\"I\",\"have\",\"a\"); value is a list of possible seen continuations, like [\"dog\",\"car\",\"cat\"] etc \n",
        "#### THIS YOU SHOULD FILL YOURSELF, I.E. FILL THE cont_dict BASED ON ngrams\n",
        "    "
      ],
      "metadata": {
        "id": "yJIGovDm-fKt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Task D\n",
        "\n",
        "* Generate new text, starting from `<bos> <bos> ...` (n-1 times) and ending after say 40 words, or `<eos>` being generated\n",
        "* I will give you a support function `sample_from` which receives a list of counts and a temperature parameter, and samples according to this distribution, returning a single column index drawn\n",
        "* The temperature sampling is described here: https://towardsdatascience.com/how-to-sample-from-language-models-682bceb97277\n",
        "* By all means, if you want to try, do try writing this function yourself!\n"
      ],
      "metadata": {
        "id": "8zL1VRNICdMi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy\n",
        "\n",
        "def softmax(x):\n",
        "    return numpy.exp(x)/sum(numpy.exp(x))\n",
        "\n",
        "def sample_from(counts,temperature=1.0):\n",
        "    \"\"\"\n",
        "    counts: list of counts that form the distribution\n",
        "    temperature: the \"how wild the generation should be\" parameter, numbers close\n",
        "                 to 0 are very conservative, numbers close or above 1 lead to quite\n",
        "                wild generations\n",
        "    \"\"\"\n",
        " \n",
        "    counts_array=numpy.array(counts)\n",
        "    #Make these sum up to 1.\n",
        "    counts_array_norm=counts_array/counts_array.sum()\n",
        "    #Divide by temperature, that is what the algorithm does\n",
        "    counts_array_norm/=temperature\n",
        "    #Renormalize into a distribution using the softmax function, that is what the algorithm does\n",
        "    final_distribution=softmax(counts_array_norm)\n",
        "    #A good way to sample from a distribution is the following function from numpy \n",
        "    x=numpy.random.multinomial(n=1,pvals=final_distribution)\n",
        "    selected_word=numpy.argmax(x).flatten()\n",
        "    return selected_word[0]\n",
        "\n",
        "sample_from([1,1,1,17],temperature=0.5) #Try running this several times each, with temps 0.1, 0.5, 1.0 ... see how temp 0.1 sticks to picking the max value, but higher temps don't?"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aD5D_DJbwc35",
        "outputId": "ed6259ec-b3c9-49ab-935d-4b6f0977ca3a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Task E: piece it all together\n",
        "\n",
        "* Again, I will give you the skeleton"
      ],
      "metadata": {
        "id": "uLL6xqnzGRnT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pprint import pprint\n",
        "\n",
        "def generate(ngrams,cont_dict,n,max_len=40,temperature=1.0,prompt=None):\n",
        "    \"\"\"\n",
        "    ngrams: the master counter\n",
        "    cont_dict: the n-1 gram continuation dict\n",
        "    n: the n in n-gram\n",
        "    max_len: how many words max?\n",
        "    temperature: the generation temperature\n",
        "    prompt: the initial prompt, as a tuple, if not given n-1 <bos> symbols will be used\n",
        "    \"\"\"\n",
        "\n",
        "    if prompt is None:\n",
        "        prompt=[\"<bos>\"]*(n-1)\n",
        "\n",
        "    generated=list(prompt) #this list will grow with words, let's initialize it with the prompt\n",
        "    for _ in range(max_len):\n",
        "        ###### HERE GENERATE THE NEXT WORD AND APPEND IT TO THE END OF `generated`\n",
        "        ###### 1. build the list of possible continuation words and their counts\n",
        "        ###### 2. sample a word from these counts\n",
        "        ###### 3. ...and append it to `generated`\n",
        "        if generated[-1]==\"<eos>\": #stop on end of sequence\n",
        "            break\n",
        "    return generated\n",
        "\n",
        "# Now we can test it!\n",
        "n=4\n",
        "for temp in (0.1,0.5,1.0,2.0,5.0):\n",
        "    generated=generate(ngrams=ngrams,cont_dict=cont_dict,n=n,max_len=60,temperature=temp)\n",
        "    print(f\"Temp={temp}:\")\n",
        "    pprint(\" \".join(generated))\n",
        "    print(\"-----------\")\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HRBDQI_1T0Hy",
        "outputId": "6c62d4b6-103b-48ae-be31-80bf68dcd8f5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Temp=0.1:\n",
            "('<bos> <bos> <bos> Difficult to tell actually who is copying whom since '\n",
            " 'apparently Deep Core is dated 2003 br br As Cosette and Valjean are riding '\n",
            " 'through the rain that fell just in front of the camera talking about '\n",
            " 'relationships in different families where it was supposed to be 2179 or '\n",
            " 'something but he never acted crazy or hot headed just the opposite')\n",
            "-----------\n",
            "Temp=0.5:\n",
            "('<bos> <bos> <bos> Camp Blood looked great when I got to buy this junk If you '\n",
            " 're researching the zombie genre br br Now over 20 years nor had she told him '\n",
            " 'about her weird father who belongs to well reputed family of Mogambo Mr '\n",
            " 'India fame Crime Master Gogo Paresh Rawal s role was pretty tame stuff and '\n",
            " 'is hit in the')\n",
            "-----------\n",
            "Temp=1.0:\n",
            "('<bos> <bos> <bos> Aw Jeez everyone here in the IMDb library And Bill Julia '\n",
            " 'and all other rentals are out or you re inviting a couple of dozen monks '\n",
            " 'live there There are times in everyones lives when people judge them for '\n",
            " 'their differences from the original Dracula himself I m a bird lover what '\n",
            " 'can I say Silly plot bad acting It')\n",
            "-----------\n",
            "Temp=2.0:\n",
            "('<bos> <bos> <bos> Plodding maybe that should be it s recommended for all '\n",
            " 'film majors <eos>')\n",
            "-----------\n",
            "Temp=5.0:\n",
            "('<bos> <bos> <bos> Gloomy Sunday is based on European literature and set in '\n",
            " 'Hawaii it s an actual fish Next time you go out of our literary folklore br '\n",
            " 'br As good as it means that Takashi Miike is a hard Right propaganda piece '\n",
            " 'much the way the driver crashes the car and possessions of the killed Poles '\n",
            " 'were Jews I personally know')\n",
            "-----------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Done!\n",
        "\n",
        "Ok, the generations are quite funny. Clearly, this is no ChatGPT, but it is also not entirely bad for a model, which is basically two dictionaries..."
      ],
      "metadata": {
        "id": "AMtAMOCJKA0z"
      }
    }
  ]
}