intro_2023_exercise_4_solution.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNAz/uelQyluV3xIWMncuql",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TurkuNLP/intro-to-nlp/blob/master/intro_2023_exercise_4_solution.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Exercise task 4: creating a dataset from corpus data (solution)\n",
        "\n",
        "This notebook shows an example solution for exercise 4.\n"
      ],
      "metadata": {
        "id": "0RscBh6fv3O_"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Setup"
      ],
      "metadata": {
        "id": "sgDc1GmswCvM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --quiet datasets"
      ],
      "metadata": {
        "id": "cfrNVDZksTnX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "24da03d7-34ca-4992-d0cf-58bd32e20e64"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m469.0/469.0 KB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.2/212.2 KB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m16.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.8/199.8 KB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.9/132.9 KB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 KB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.2/114.2 KB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m264.6/264.6 KB\u001b[0m \u001b[31m7.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m158.8/158.8 KB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "\n",
        "import datasets"
      ],
      "metadata": {
        "id": "p9SqpAiCFFX3"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Process data into dataset"
      ],
      "metadata": {
        "id": "G-RxtZUNw3Ew"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Download raw data from given URLs to local drive"
      ],
      "metadata": {
        "id": "79xGo35Nxnu7"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "9r2Fd-dsks9A"
      },
      "outputs": [],
      "source": [
        "!wget --quiet -nc http://dl.turkunlp.org/TKO_7095_2023/imdb-positives.txt\n",
        "!wget --quiet -nc http://dl.turkunlp.org/TKO_7095_2023/imdb-negatives.txt"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Read in one text per line format"
      ],
      "metadata": {
        "id": "8laPFhDOxtbl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "positive_texts = open('imdb-positives.txt').readlines()\n",
        "negative_texts = open('imdb-negatives.txt').readlines()"
      ],
      "metadata": {
        "id": "bR_xONuqk14p"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Reformat as list of dictionaries with `text` and `label`"
      ],
      "metadata": {
        "id": "cJNXXJkNx7H9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "positive_examples = [{ 'text': text, 'label': 'positive' } for text in positive_texts]\n",
        "negative_examples = [{ 'text': text, 'label': 'negative' } for text in negative_texts]"
      ],
      "metadata": {
        "id": "pUM_F9cjlFSF"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Combine these"
      ],
      "metadata": {
        "id": "JbpuwzMZyIvb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "all_examples = positive_examples + negative_examples"
      ],
      "metadata": {
        "id": "4pqpP0ydlWtk"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Important**! Shuffle your data so that you don't have all positives first followed by all negatives"
      ],
      "metadata": {
        "id": "3un3DMB0yNYo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "random.shuffle(all_examples)"
      ],
      "metadata": {
        "id": "g6Mf_Peklenu"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Split into train, validation and test sets"
      ],
      "metadata": {
        "id": "fN0QoW77yVeR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "total_size = len(all_examples)\n",
        "\n",
        "train_size = int(0.8*total_size)\n",
        "valid_size = int(0.1*total_size)\n",
        "test_size = total_size - train_size - valid_size\n",
        "\n",
        "train_examples = all_examples[:train_size]\n",
        "valid_examples = all_examples[train_size:train_size+valid_size]\n",
        "test_examples = all_examples[train_size+valid_size:]\n",
        "\n",
        "assert all_examples == train_examples + valid_examples + test_examples"
      ],
      "metadata": {
        "id": "Rj0W1l8olhuS"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Reformat as datasets. Notice that `Dataset.from_dict` expects a dictionary of lists rather than a list of dictionaries."
      ],
      "metadata": {
        "id": "J5anuyKlyZHU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def make_dataset(examples):\n",
        "  texts, labels = [], []\n",
        "  for e in examples:\n",
        "    texts.append(e['text'])\n",
        "    labels.append(e['label'])\n",
        "  data = {\n",
        "      'text': texts,\n",
        "      'label': labels      \n",
        "  }\n",
        "  return datasets.Dataset.from_dict(data)\n",
        "\n",
        "\n",
        "train = make_dataset(train_examples)\n",
        "valid = make_dataset(valid_examples)\n",
        "test = make_dataset(test_examples)"
      ],
      "metadata": {
        "id": "WYPqGBOwEwhK"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Create `DatasetDict` to wrap the `Dataset` objects"
      ],
      "metadata": {
        "id": "UIxmiuD0ylMq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data = {\n",
        "    'train': train,\n",
        "    'validation': valid,\n",
        "    'test': test,    \n",
        "}\n",
        "\n",
        "dataset = datasets.DatasetDict(data)"
      ],
      "metadata": {
        "id": "j4LxMK78FUNL"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Check final dataset"
      ],
      "metadata": {
        "id": "VlYmdv3Syp_2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nHXUojOmFs3s",
        "outputId": "0c08f29b-3cab-4d8b-c2d2-ffffb39d290f"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['text', 'label'],\n",
              "        num_rows: 40000\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['text', 'label'],\n",
              "        num_rows: 5000\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['text', 'label'],\n",
              "        num_rows: 5000\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    }
  ]
}