ex9_intro_to_hlt_2023_v2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "6cb8849da4eb4de5834cae0da4ac81ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_64c545796eab4071903ef80275e2b2d6",
              "IPY_MODEL_729e5eac79df4812a01397dbd3d3e22d",
              "IPY_MODEL_9b29d52a6b734c73ac8362c36fd8dd48"
            ],
            "layout": "IPY_MODEL_6ccc79e5d40d4438baf86cb38b2705af"
          }
        },
        "64c545796eab4071903ef80275e2b2d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_696fae99698740249a27c31e20303c60",
            "placeholder": "​",
            "style": "IPY_MODEL_c14be0435fd84bdb8991b654c81b3045",
            "value": "100%"
          }
        },
        "729e5eac79df4812a01397dbd3d3e22d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9dfb87ed187143f0b6ca59d3fffd7224",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5cda0e27d1b6445b9b713e6eccd46012",
            "value": 3
          }
        },
        "9b29d52a6b734c73ac8362c36fd8dd48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_21c661ddcf174e599811a3cd9513ca11",
            "placeholder": "​",
            "style": "IPY_MODEL_01a4d87ea5a54f16bbeae72894c85632",
            "value": " 3/3 [00:00&lt;00:00, 84.16it/s]"
          }
        },
        "6ccc79e5d40d4438baf86cb38b2705af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "696fae99698740249a27c31e20303c60": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c14be0435fd84bdb8991b654c81b3045": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9dfb87ed187143f0b6ca59d3fffd7224": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5cda0e27d1b6445b9b713e6eccd46012": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "21c661ddcf174e599811a3cd9513ca11": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "01a4d87ea5a54f16bbeae72894c85632": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TurkuNLP/intro-to-nlp/blob/master/ex9_intro_to_hlt_2023_v2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Watch out, this notebook stretches colab memory with n=5, so you might need to \"Restart and run all\" on full re-runs of the notebook,\n",
        "since the old data clogs the memory during a rerun**\n",
        "\n",
        "In this exercise, you'll try to generate text with an n-gram model. In the generation, we use the last generated n-1 words as the prefix, and the n-gram counts to establish the distribution of possible continuations. So we might run this off the following data structure:\n",
        "\n",
        "* A master dictionary, where the key are n-1 grams\n",
        "* The value is another dictionary\n",
        "* In this dictonary the key is a word\n",
        "* And the value is its count\n",
        "\n",
        "So, when generating, we can take the last n-1 words, look them up in the master dictionary, and we get a dictionary of all seen continuations and their counts.\n",
        "\n",
        "Let us divide it to the following tasks:\n",
        "\n",
        "1. Generate n-grams from a corpus of text, e.g. the IMDB dataset\n",
        "2. Count the n-grams, i.e. build the master dictionary\n",
        "\n",
        "With these data structures, the generation can proceed quite easily. Say, we have a 4-gram model.\n",
        "\n",
        "* Given a prior context $w_1w_2w_3$\n",
        "* Look up the word-count dictionary of possible words $w_4$\n",
        "* The counts, once normalized to sum up to 1, form a distribution over words that can continue $w_1w_2w_3$ and we can sample the next word from this distribution.\n",
        "* Then we append this generated word to our list of already generated words, and repeat the process\n",
        "\n",
        "\n",
        "Other remarks:\n",
        "\n",
        "* We want to pad all texts with `<bos>` (beginning of sequence) and <eos> (end of sequence). The `<bos>` we want to have there n-1 times, so we can use it as the initial prompt and let the model learn how the sequences start. The `<eos>` allows us to stop generating, and prevents a crash on unknown n-grams at the very end of a sequence. (if an n-gram $w_1w_2w_3w_4$ was seen only once at the end of a \"training\" sequence, then an attempt to continue it during generation, would lead to a crash, since we have no known n-gram to continue the sequence $w_2w_3w_4$ with our simple, unsmoothed model :)\n"
      ],
      "metadata": {
        "id": "nF1-YWS82oKK"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Task A: Generate n-grams\n",
        "\n",
        "* Write a generator function (using `yield` rather than `return`) which yields n-grams as tuples $(w_1,...,w_n)$ from all sections of the IMDB dataset\n",
        "* a vectorizer from `sklearn` can be used as a trivial tokenizer\n",
        "* `more-itertools` is a nifty library to achieve the n-gram generation\n",
        "* remember to pad with n-1 `<bos>` symbols at the beginning, and one `<eos>` symbol at the end\n",
        "\n",
        "You can give this a shot, or simply use the code below."
      ],
      "metadata": {
        "id": "qD9E2rDD7oyE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip3 install datasets more-itertools"
      ],
      "metadata": {
        "id": "a4efOx1BpIN9",
        "outputId": "88b0dbdd-9c55-4c04-bd5e-8f34f936a48f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.9/dist-packages (2.11.0)\n",
            "Requirement already satisfied: more-itertools in /usr/local/lib/python3.9/dist-packages (9.1.0)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (9.0.0)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.9/dist-packages (from datasets) (3.2.0)\n",
            "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.3.6)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.9/dist-packages (from datasets) (0.70.14)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.9/dist-packages (from datasets) (3.8.4)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.9/dist-packages (from datasets) (23.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.11.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.13.4)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (4.65.0)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (2023.3.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (6.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from datasets) (1.22.4)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (2.27.1)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.18.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (22.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.3.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (2.0.12)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.8.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (3.11.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (1.26.15)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (3.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (2022.12.7)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2022.7.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WklQqyCFo_aq"
      },
      "outputs": [],
      "source": [
        "import datasets\n",
        "import sklearn.feature_extraction"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dset=datasets.load_dataset(\"imdb\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86,
          "referenced_widgets": [
            "6cb8849da4eb4de5834cae0da4ac81ba",
            "64c545796eab4071903ef80275e2b2d6",
            "729e5eac79df4812a01397dbd3d3e22d",
            "9b29d52a6b734c73ac8362c36fd8dd48",
            "6ccc79e5d40d4438baf86cb38b2705af",
            "696fae99698740249a27c31e20303c60",
            "c14be0435fd84bdb8991b654c81b3045",
            "9dfb87ed187143f0b6ca59d3fffd7224",
            "5cda0e27d1b6445b9b713e6eccd46012",
            "21c661ddcf174e599811a3cd9513ca11",
            "01a4d87ea5a54f16bbeae72894c85632"
          ]
        },
        "id": "BAiGkrtqpYo1",
        "outputId": "cd530b6b-6d4f-4124-86b1-b59d103f911b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.builder:Found cached dataset imdb (/root/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6cb8849da4eb4de5834cae0da4ac81ba"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Few remarks here:\n",
        "# 1. we don't need the vectorizer per se, we just want its analyzer function, which basically tokenizes the text for us, and somewhat unfortunately drops punctuation\n",
        "# 2. the default token pattern in sklearn drops 1-letter words (like \"I\" and \"a\") so I modify it a bit\n",
        "# 3. it's a pretty lousy tokenizer, but it will do for this toy exercise\n",
        "cvectorizer=sklearn.feature_extraction.text.CountVectorizer(lowercase=False,stop_words=None,token_pattern=r\"(?u)\\b\\w+\\b\" )\n",
        "analyzer=cvectorizer.build_analyzer()\n",
        "analyzer(\"I have a dog at home, it likes to shred newspapers.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HeKF44p4pjyE",
        "outputId": "c126fc77-e32c-4df1-ba26-71522fe5e9ec"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['I',\n",
              " 'have',\n",
              " 'a',\n",
              " 'dog',\n",
              " 'at',\n",
              " 'home',\n",
              " 'it',\n",
              " 'likes',\n",
              " 'to',\n",
              " 'shred',\n",
              " 'newspapers']"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Now we tokenize the IMDB dataset the usual way\n",
        "def tokenize(ex):\n",
        "    return {\"tokenized\":analyzer(ex[\"text\"])}\n",
        "\n",
        "dset=dset.map(tokenize,num_proc=4)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eCdn2w-drcVN",
        "outputId": "04c4d74a-213d-405b-fce6-357f832ef130"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0/cache-d09f1293d5219067_*_of_00004.arrow\n",
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0/cache-575cdb577c85a33d_*_of_00004.arrow\n",
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0/cache-9cafbb586a726924_*_of_00004.arrow\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from collections import Counter\n",
        "from more_itertools import sliding_window #more-itertools is an awesome library!\n",
        "import tqdm\n",
        "\n",
        "def generate_ngrams(dset,n):\n",
        "    for ex in tqdm.tqdm(dset):\n",
        "        tokens=[\"<bos>\"]*(n-1)+ex[\"tokenized\"]+[\"<eos>\"]\n",
        "        for ngram in sliding_window(tokens,n):\n",
        "            yield ngram\n",
        "\n"
      ],
      "metadata": {
        "id": "OPXGteLQtOXD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Task B\n",
        "\n",
        "* Now we can combine the different sections of the IMDB dataset and count our n-grams\n"
      ],
      "metadata": {
        "id": "LOoapsbtAcyM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Here we can concatenate all the individual datasets (train,test,unlabeled) in IMDB\n",
        "# the \"master\" dataset is a dictionary of these, so dset.values() has the datasets of the individual sections (train,test,unlabeled)\n",
        "combined_dataset=datasets.concatenate_datasets(list(dset.values()))\n"
      ],
      "metadata": {
        "id": "UiuqC8yl_lqo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ngrams={} #This is the master dictionary\n",
        "for ngram in generate_ngrams(combined_dataset,4): #let's start with 4-grams, you can try 2- 3- and 5- grams too!\n",
        "    #### YOU NEED TO FILL CODE HERE :)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B4qumygtAYZH",
        "outputId": "de4e8a47-1500-44d3-e749-0471fc1b7c23"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 100000/100000 [01:36<00:00, 1037.94it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Task C\n",
        "\n",
        "* Generate new text, starting from `<bos> <bos> ...` (n-1 times) and ending after say 40 words, or `<eos>` being generated\n",
        "* I will give you a support function `sample_from` which receives a list of counts and a temperature parameter, and samples according to this distribution, returning a single column index drawn\n",
        "* The temperature sampling is described here: https://towardsdatascience.com/how-to-sample-from-language-models-682bceb97277\n",
        "* By all means, if you want to try, do try writing this function yourself!\n"
      ],
      "metadata": {
        "id": "8zL1VRNICdMi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy\n",
        "\n",
        "def softmax(x):\n",
        "    return numpy.exp(x)/sum(numpy.exp(x))\n",
        "\n",
        "def sample_from(counts,temperature=1.0):\n",
        "    \"\"\"\n",
        "    counts: list of counts that form the distribution\n",
        "    temperature: the \"how wild the generation should be\" parameter, numbers close\n",
        "                 to 0 are very conservative, numbers close or above 1 lead to quite\n",
        "                wild generations\n",
        "    \"\"\"\n",
        "\n",
        "    counts_array=numpy.array(counts)\n",
        "    #Make these sum up to 1.\n",
        "    counts_array_norm=counts_array/counts_array.sum()\n",
        "    #Divide by temperature, that is what the algorithm does\n",
        "    counts_array_norm/=temperature\n",
        "    #Renormalize into a distribution using the softmax function, that is what the algorithm does\n",
        "    final_distribution=softmax(counts_array_norm)\n",
        "    #A good way to sample from a distribution is the following function from numpy\n",
        "    x=numpy.random.multinomial(n=1,pvals=final_distribution)\n",
        "    selected_word=numpy.argmax(x).flatten()\n",
        "    return selected_word[0]\n",
        "\n",
        "sample_from([1,1,1,17],temperature=0.5) #Try running this several times each, with temps 0.1, 0.5, 1.0 ... see how temp 0.1 sticks to picking the max value, but higher temps don't?"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aD5D_DJbwc35",
        "outputId": "2a89022f-0db1-4a55-956f-ffebf77c4724"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Task D: piece it all together\n",
        "\n",
        "* Again, I will give you the skeleton"
      ],
      "metadata": {
        "id": "uLL6xqnzGRnT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pprint import pprint\n",
        "\n",
        "def generate(ngrams,n,max_len=40,temperature=1.0,prompt=None):\n",
        "    \"\"\"\n",
        "    ngrams: the master dictionary\n",
        "    n: the n in n-gram\n",
        "    max_len: how many words max?\n",
        "    temperature: the generation temperature\n",
        "    prompt: the initial prompt, as a tuple, if not given n-1 <bos> symbols will be used\n",
        "    \"\"\"\n",
        "\n",
        "    if prompt is None:\n",
        "        prompt=[\"<bos>\"]*(n-1)\n",
        "\n",
        "    generated=list(prompt) #this list will grow with words\n",
        "    for _ in range(max_len):\n",
        "        ##### HERE GOES THE CODE TO GENERATE THE NEXT WORD\n",
        "\n",
        "        if generated[-1]==\"<eos>\": #stop on end of sequence\n",
        "            break\n",
        "    return generated\n",
        "\n",
        "# Now we can test it!\n",
        "\n",
        "# make sure to match the n below to the n which was used to create\n",
        "# the master dictionary\n",
        "for temp in (0.1,0.5,1.0,2.0,5.0):\n",
        "    generated=generate(ngrams=ngrams,n=4,max_len=60,temperature=temp)\n",
        "    print(f\"Temp={temp}:\")\n",
        "    pprint(\" \".join(generated))\n",
        "    print(\"-----------\")\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HRBDQI_1T0Hy",
        "outputId": "58759639-c770-4ecf-9e65-12c602320020"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Temp=0.1:\n",
            "('<bos> <bos> <bos> <bos> Ava Gardner Kathryn Grayson Howard Keel Marge Gower '\n",
            " 'Champion all under the sure and competent direction of George Sidney it had '\n",
            " 'the benefits of a pleasant score and best of all as the bridge starts '\n",
            " 'creaking rather than try her best to run across she just stands there and '\n",
            " 'WALKS away He WALKS AWAY like a little boy during her')\n",
            "-----------\n",
            "Temp=0.5:\n",
            "('<bos> <bos> <bos> <bos> Audiences will groan at the flimsy attempts at humor '\n",
            " 'the awkward camera work the sexism and racism here is despicable Were Bob s '\n",
            " 'and Bing s egos that fragile or that inflated I couldn t decide which one '\n",
            " 'Lines such as Yeah I m so bad I kick my own ass twice a day call for further '\n",
            " 'investigation Either way')\n",
            "-----------\n",
            "Temp=1.0:\n",
            "('<bos> <bos> <bos> <bos> Crackerjack is a simple but drawn out Harlequin '\n",
            " 'romance about doomed lovers constantly separated by events beyond their '\n",
            " 'control where the biggest surprise is that Fabio doesn t turn up The bride '\n",
            " 'that is Now in a deep spiral of depression our hero doesn t see any way this '\n",
            " 'movie could be watched without realizing it was by Carlos R')\n",
            "-----------\n",
            "Temp=2.0:\n",
            "('<bos> <bos> <bos> <bos> Girlfriends is an EXCELLENT show which expresses how '\n",
            " 'some friends can truly help to change your life it s fun and camp in a butch '\n",
            " 'kinda was these guys can act and they over enunciate EVERYTHING imagine a '\n",
            " 'movie cast entirely with the Harry Potter like boarding school segments How '\n",
            " 'did the interview with the first girl crush who now')\n",
            "-----------\n",
            "Temp=5.0:\n",
            "('<bos> <bos> <bos> <bos> CHUD only good point is gore which is not limited '\n",
            " 'that s why I find this movie so anti american Sure there are moments when '\n",
            " 'you just want to sit laugh cry and then pee I m serious Don t watch this '\n",
            " 'boring junk <eos>')\n",
            "-----------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Done!\n",
        "\n",
        "Ok, the generations are quite funny. Clearly, this is no ChatGPT, but it is also not entirely bad for a model, which is basically two dictionaries..."
      ],
      "metadata": {
        "id": "AMtAMOCJKA0z"
      }
    }
  ]
}