7.PRETRAIN_METHOD/7.4.2.gpt2_finetune_NSMC.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import tensorflow as tf\n",
    "from transformers import TFGPT2Model\n",
    "\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "import gluonnlp as nlp\n",
    "from gluonnlp.data import SentencepieceTokenizer\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "아레 실행 커멘드는 gpt_ckpt 폴더가 있지 않은 경우에만 실행해주세요."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import wget\n",
    "import zipfile\n",
    "\n",
    "wget.download('https://github.com/NLP-kr/tensorflow-ml-nlp-tf2/releases/download/v1.0/gpt_ckpt.zip')\n",
    "\n",
    "with zipfile.ZipFile('gpt_ckpt.zip') as z:\n",
    "    z.extractall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 시각화\n",
    "\n",
    "def plot_graphs(history, string):\n",
    "    plt.plot(history.history[string])\n",
    "    plt.plot(history.history['val_'+string], '')\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(string)\n",
    "    plt.legend([string, 'val_'+string])\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED_NUM = 1234\n",
    "tf.random.set_seed(SEED_NUM)\n",
    "np.random.seed(SEED_NUM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 데이터 준비하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TOKENIZER_PATH = './gpt_ckpt/gpt2_kor_tokenizer.spiece'\n",
    "\n",
    "tokenizer = SentencepieceTokenizer(TOKENIZER_PATH)\n",
    "vocab = nlp.vocab.BERTVocab.from_sentencepiece(TOKENIZER_PATH,\n",
    "                                               mask_token=None,\n",
    "                                               sep_token='<unused0>',\n",
    "                                               cls_token=None,\n",
    "                                               unknown_token='<unk>',\n",
    "                                               padding_token='<pad>',\n",
    "                                               bos_token='<s>',\n",
    "                                               eos_token='</s>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "NUM_EPOCHS = 3\n",
    "VALID_SPLIT = 0.1\n",
    "SENT_MAX_LEN = 39\n",
    "\n",
    "DATA_IN_PATH = './data_in/KOR'\n",
    "DATA_OUT_PATH = \"./data_out/KOR\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 데이터 전처리 준비\n",
    "\n",
    "DATA_TRAIN_PATH = os.path.join(DATA_IN_PATH, \"naver_movie\", \"ratings_train.txt\")\n",
    "DATA_TEST_PATH = os.path.join(DATA_IN_PATH, \"naver_movie\", \"ratings_test.txt\")\n",
    "\n",
    "train_data = pd.read_csv(DATA_TRAIN_PATH, header = 0, delimiter = '\\t', quoting = 3)\n",
    "train_data = train_data.dropna()\n",
    "train_data.head()\n",
    "\n",
    "print(\"Total # dataset: train - {}\".format(len(train_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 텍스트 전처리\n",
    "\n",
    "def clean_text(sent):\n",
    "    sent_clean = re.sub(\"[^가-힣ㄱ-ㅎㅏ-ㅣ\\\\s]\", \"\", sent)\n",
    "    return sent_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_data = train_data[:50] # for test\n",
    "\n",
    "train_data_sents = []\n",
    "train_data_labels = []\n",
    "\n",
    "for train_sent, train_label in train_data[['document', 'label']].values:\n",
    "    train_tokenized_text = vocab[tokenizer(clean_text(train_sent))]\n",
    "\n",
    "    tokens = [vocab[vocab.bos_token]]  \n",
    "    tokens += pad_sequences([train_tokenized_text], \n",
    "                            SENT_MAX_LEN, \n",
    "                            value=vocab[vocab.padding_token], \n",
    "                            padding='post').tolist()[0] \n",
    "    tokens += [vocab[vocab.eos_token]]\n",
    "\n",
    "    train_data_sents.append(tokens)\n",
    "    train_data_labels.append(train_label)\n",
    "\n",
    "train_data_sents = np.array(train_data_sents, dtype=np.int64)\n",
    "train_data_labels = np.array(train_data_labels, dtype=np.int64)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 모델 학습"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TFGPT2Classifier(tf.keras.Model):\n",
    "    def __init__(self, dir_path, num_class):\n",
    "        super(TFGPT2Classifier, self).__init__()\n",
    "        \n",
    "        self.gpt2 = TFGPT2Model.from_pretrained(dir_path)\n",
    "        self.num_class = num_class\n",
    "        \n",
    "        self.dropout = tf.keras.layers.Dropout(self.gpt2.config.summary_first_dropout)\n",
    "        self.classifier = tf.keras.layers.Dense(self.num_class, \n",
    "                                                kernel_initializer=tf.keras.initializers.TruncatedNormal(stddev=self.gpt2.config.initializer_range), \n",
    "                                                name=\"classifier\")\n",
    "        \n",
    "    def call(self, inputs):\n",
    "        outputs = self.gpt2(inputs)\n",
    "        pooled_output = outputs[0][:, -1]\n",
    "\n",
    "        pooled_output = self.dropout(pooled_output)\n",
    "        logits = self.classifier(pooled_output)\n",
    "\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_MODEL_PATH = './gpt_ckpt'\n",
    "cls_model = TFGPT2Classifier(dir_path=BASE_MODEL_PATH, num_class=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.keras.optimizers.Adam(learning_rate=6.25e-5)\n",
    "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')\n",
    "cls_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"tf2_gpt2_naver_movie\"\n",
    "\n",
    "earlystop_callback = EarlyStopping(monitor='val_accuracy', min_delta=0.0001, patience=2)\n",
    "\n",
    "checkpoint_path = os.path.join(DATA_OUT_PATH, model_name, 'weights.h5')\n",
    "checkpoint_dir = os.path.dirname(checkpoint_path)\n",
    "\n",
    "if os.path.exists(checkpoint_dir):\n",
    "    print(\"{} -- Folder already exists \\n\".format(checkpoint_dir))\n",
    "else:\n",
    "    os.makedirs(checkpoint_dir, exist_ok=True)\n",
    "    print(\"{} -- Folder create complete \\n\".format(checkpoint_dir))\n",
    "    \n",
    "cp_callback = ModelCheckpoint(\n",
    "    checkpoint_path, monitor='val_accuracy', verbose=1, save_best_only=True, save_weights_only=True)\n",
    "\n",
    "history = cls_model.fit(train_data_sents, train_data_labels, \n",
    "                        epochs=NUM_EPOCHS, \n",
    "                        batch_size=BATCH_SIZE,\n",
    "                        validation_split=VALID_SPLIT, \n",
    "                        callbacks=[earlystop_callback, cp_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_graphs(history, 'accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_graphs(history, 'loss')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 모델 테스트"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = pd.read_csv(DATA_TEST_PATH, header=0, delimiter='\\t', quoting=3)\n",
    "test_data = test_data.dropna()\n",
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_data = test_data[:50] # for test\n",
    "\n",
    "test_data_sents = []\n",
    "test_data_labels = []\n",
    "\n",
    "for test_sent, test_label in test_data[['document','label']].values:\n",
    "    test_tokenized_text = vocab[tokenizer(clean_text(test_sent))]\n",
    "\n",
    "    tokens = [vocab[vocab.bos_token]]  \n",
    "    tokens += pad_sequences([test_tokenized_text], \n",
    "                            SENT_MAX_LEN, \n",
    "                            value=vocab[vocab.padding_token], \n",
    "                            padding='post').tolist()[0] \n",
    "    tokens += [vocab[vocab.eos_token]]\n",
    "\n",
    "    test_data_sents.append(tokens)\n",
    "    test_data_labels.append(test_label)\n",
    "\n",
    "test_data_sents = np.array(test_data_sents, dtype=np.int64)\n",
    "test_data_labels = np.array(test_data_labels, dtype=np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"num sents, labels {}, {}\".format(len(test_data_sents), len(test_data_labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cls_model.load_weights(checkpoint_path)\n",
    "\n",
    "results = cls_model.evaluate(test_data_sents, test_data_labels, batch_size=1024)\n",
    "print(\"test loss, test acc: \", results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}