linguistic_tenseflow.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y4k0uw-d0vTA"
      },
      "outputs": [],
      "source": [
        "!pip install transformers\n",
        "!pip install datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2Ni5e2vx04R_"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "\n",
        "from transformers import RobertaTokenizer, TFRobertaModel\n",
        "from transformers import BertTokenizer, TFBertModel\n",
        "from transformers import XLNetTokenizer, TFXLNetModel\n",
        "from datasets import load_dataset\n",
        "from huggingface_hub import login\n",
        "from sklearn.utils import shuffle\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "import os\n",
        "import pandas as pd\n",
        "import regex as re\n",
        "import string"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qTk7Wxjp14JW"
      },
      "outputs": [],
      "source": [
        "dataset = load_dataset(\"nevikw39/ADReSSo_whisper-large-v3_transcript\")\n",
        "dataset[\"train\"]=dataset[\"train\"].shuffle(seed=42)\n",
        "transcript = dataset[\"train\"]['transcript_no-chunked']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xjJbCbmz1Ep4"
      },
      "outputs": [],
      "source": [
        "models = {'roberta':(RobertaTokenizer, 'roberta-large', TFRobertaModel),\n",
        "          'bert':(BertTokenizer, 'bert-base-uncased', TFBertModel),\n",
        "          'xlnet':(XLNetTokenizer, 'xlnet-large-cased', TFXLNetModel)}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "COL3gI1G1HbI"
      },
      "outputs": [],
      "source": [
        "tokenizer, model_type, model_name = models['roberta']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UNomHjBv1MO3"
      },
      "outputs": [],
      "source": [
        "def make_inputs(tokenizer, model_type, serie, max_len = 256):\n",
        "    tokenizer = tokenizer.from_pretrained(model_type, lowercase=True )\n",
        "    tokenized_data = [tokenizer.encode_plus(text, max_length=max_len,\n",
        "                                            padding = 'max_length',\n",
        "                                            add_special_tokens=True,\n",
        "                                            truncation = True) for text in serie]\n",
        "\n",
        "    input_ids = np.array([text['input_ids'] for text in tokenized_data])\n",
        "    attention_mask = np.array([text['attention_mask'] for text in tokenized_data])\n",
        "    return input_ids, attention_mask"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7d8N0mjB1OhM"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "input_ids_train, attention_mask_train = \\\n",
        "make_inputs(tokenizer, model_type, transcript, max_len = 256)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lyZ3oP1F1RRU"
      },
      "outputs": [],
      "source": [
        "##### TPU or no TPU\n",
        "def init_model(model_name, model_type, num_labels, Tpu = 'on', max_len = 256):\n",
        "# ------------------------------------------------ with TPU --------------------------------------------------------------#\n",
        "    if Tpu == 'on':\n",
        "        # a few lines of code to get our tpu started and our data distributed on it\n",
        "        resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])\n",
        "        tf.config.experimental_connect_to_cluster(resolver)\n",
        "        tf.tpu.experimental.initialize_tpu_system(resolver)\n",
        "        # print(\"All devices: \", tf.config.list_logical_devices('TPU'))\n",
        "\n",
        "        strategy = tf.distribute.TPUStrategy(resolver)\n",
        "        with strategy.scope():\n",
        "\n",
        "            model_ = model_name.from_pretrained(model_type)\n",
        "            # inputs\n",
        "            input_ids = tf.keras.Input(shape = (max_len, ), dtype = 'int32')\n",
        "            attention_masks = tf.keras.Input(shape = (max_len,), dtype = 'int32')\n",
        "\n",
        "            outputs = model_([input_ids, attention_masks])\n",
        "\n",
        "            if 'xlnet' in model_type:\n",
        "                # cls is the last token in xlnet tokenization\n",
        "                outputs = outputs[0]\n",
        "                cls_output = tf.squeeze(outputs[:, -1:, :], axis=1)\n",
        "            else:\n",
        "                cls_output = outputs[1]\n",
        "\n",
        "            final_output = tf.keras.layers.Dense(num_labels, activation = 'softmax')(cls_output)\n",
        "            model = tf.keras.Model(inputs = [input_ids, attention_masks], outputs = final_output)\n",
        "            model.compile(optimizer = Adam(learning_rate = 5e-6), loss = 'categorical_crossentropy',\n",
        "                        metrics = ['accuracy'])\n",
        "# ------------------------------------------------ without TPU --------------------------------------------------------------#\n",
        "    else:\n",
        "        model_ = model_name.from_pretrained(model_type)\n",
        "        # inputs\n",
        "        input_ids = tf.keras.Input(shape = (max_len, ), dtype = 'int32')\n",
        "        attention_masks = tf.keras.Input(shape = (max_len,), dtype = 'int32')\n",
        "\n",
        "        outputs = model_([input_ids, attention_masks])\n",
        "\n",
        "        if 'xlnet' in model_type:\n",
        "            # cls is the last token in xlnet tokenization\n",
        "            outputs = outputs[0]\n",
        "            cls_output = tf.squeeze(outputs[:, -1:, :], axis=1)\n",
        "        else:\n",
        "            cls_output = outputs[1]\n",
        "\n",
        "\n",
        "        final_output = tf.keras.layers.Dense(num_labels, activation = 'softmax')(cls_output)\n",
        "\n",
        "        model = tf.keras.Model(inputs = [input_ids, attention_masks], outputs = final_output)\n",
        "\n",
        "        model.compile(optimizer = Adam(learning_rate = 5e-6), loss = 'binary_crossentropy',\n",
        "                    metrics = ['accuracy'])\n",
        "    return model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wrvgTLii1UXt"
      },
      "outputs": [],
      "source": [
        "model = init_model(model_name, model_type, num_labels = 2, Tpu = 'on', max_len = 256)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sLNAGqDs1Zz-"
      },
      "outputs": [],
      "source": [
        "train_y = tf.keras.utils.to_categorical(dataset[\"train\"][\"label\"], num_classes=2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "JeIrT8Ct1ePe"
      },
      "outputs": [],
      "source": [
        "history = model.fit([input_ids_train, attention_mask_train], train_y,\n",
        "          callbacks=tf.keras.callbacks.EarlyStopping(patience=10),\n",
        "          validation_split=0.25, epochs = 50, batch_size = 2,\n",
        "          shuffle = True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "s-YHJbiI_6YB"
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "9_K83cIcKbg5"
      },
      "outputs": [],
      "source": [
        "input_ids_test, attention_mask_test = make_inputs(tokenizer, model_type, dataset[\"test\"]['transcript_no-chunked'], max_len = 256)\n",
        "model.evaluate(x = [input_ids_test, attention_mask_test],\n",
        "               y = tf.keras.utils.to_categorical(dataset[\"test\"][\"label\"], num_classes=2),\n",
        "               batch_size = 4)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}