From 588d8ef9cf2256cf1a16223f7f37b47b52ac529e Mon Sep 17 00:00:00 2001
From: jcollopy-tulane <jcollopy@tulane.edu>
Date: Tue, 30 Apr 2024 10:01:53 -0500
Subject: [PATCH] Verbose

---
 notebooks/New_Bert.ipynb | 165 +++++++++------------------------------
 1 file changed, 35 insertions(+), 130 deletions(-)

diff --git a/notebooks/New_Bert.ipynb b/notebooks/New_Bert.ipynb
index c2b76cb..d9b636a 100644
--- a/notebooks/New_Bert.ipynb
+++ b/notebooks/New_Bert.ipynb
@@ -1,5 +1,21 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "d9210791-41e0-42f3-99b4-a93439662152",
+   "metadata": {},
+   "source": [
+    "### BERT Experiment"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d36e1d24-9ab5-41e7-be19-54b2d319363c",
+   "metadata": {},
+   "source": [
+    "Below, the necessary libaries are imported and data uploaded."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 39,
@@ -33,6 +49,14 @@
     "test_df = pd.read_csv(\"../data/test.csv\")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "24896d87-bc73-4cf4-a0dd-ee788685f056",
+   "metadata": {},
+   "source": [
+    "Below the MiniBERT tokenizer is initialized. Then, a custom dataset class is created that will create the necessary datasets that are compatible with PyTorch."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 41,
@@ -77,139 +101,20 @@
     "test_loader = DataLoader(test_dataset, batch_size=10)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "121b1db7-6173-4310-bdf5-6836a5cfe0e3",
+   "metadata": {},
+   "source": [
+    "Below, the training of the model begins. Training and validation accuracy are accounted for and plotted as well."
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": 42,
-   "id": "3a7f59a5-371e-4c5c-9c6c-6a60448bf365",
+   "execution_count": null,
+   "id": "316b3c35-3982-46a4-a3ab-62da1203d6ac",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at prajjwal1/bert-mini and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
-      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
-      "/Users/jackiecollopy/nlp-virtual/lib/python3.11/site-packages/transformers/optimization.py:521: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
-      "  warnings.warn(\n",
-      "Epoch 1/6 - Training:   0%|                                                      | 0/683 [00:00<?, ?it/s]/var/folders/hs/br_4rpdj68nc3sfdpgv0xgn80000gn/T/ipykernel_12767/33828928.py:18: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
-      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
-      "Epoch 1/6 - Training: 100%|████████████████████████████████████████████| 683/683 [01:15<00:00,  9.03it/s]\n",
-      "Epoch 1/6 - Validation:   0%|                                                    | 0/228 [00:00<?, ?it/s]/var/folders/hs/br_4rpdj68nc3sfdpgv0xgn80000gn/T/ipykernel_12767/33828928.py:18: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
-      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
-      "Epoch 1/6 - Validation: 100%|██████████████████████████████████████████| 228/228 [00:07<00:00, 31.91it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Epoch 1/6 - Training Loss: 0.6837, Training Accuracy: 0.5511, Validation Accuracy: 0.6046\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Epoch 2/6 - Training:   0%|                                                      | 0/683 [00:00<?, ?it/s]/var/folders/hs/br_4rpdj68nc3sfdpgv0xgn80000gn/T/ipykernel_12767/33828928.py:18: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
-      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
-      "Epoch 2/6 - Training: 100%|████████████████████████████████████████████| 683/683 [01:20<00:00,  8.49it/s]\n",
-      "Epoch 2/6 - Validation:   0%|                                                    | 0/228 [00:00<?, ?it/s]/var/folders/hs/br_4rpdj68nc3sfdpgv0xgn80000gn/T/ipykernel_12767/33828928.py:18: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
-      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
-      "Epoch 2/6 - Validation: 100%|██████████████████████████████████████████| 228/228 [00:07<00:00, 30.44it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Epoch 2/6 - Training Loss: 0.6428, Training Accuracy: 0.6295, Validation Accuracy: 0.6428\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Epoch 3/6 - Training:   0%|                                                      | 0/683 [00:00<?, ?it/s]/var/folders/hs/br_4rpdj68nc3sfdpgv0xgn80000gn/T/ipykernel_12767/33828928.py:18: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
-      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
-      "Epoch 3/6 - Training: 100%|████████████████████████████████████████████| 683/683 [01:20<00:00,  8.51it/s]\n",
-      "Epoch 3/6 - Validation:   0%|                                                    | 0/228 [00:00<?, ?it/s]/var/folders/hs/br_4rpdj68nc3sfdpgv0xgn80000gn/T/ipykernel_12767/33828928.py:18: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
-      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
-      "Epoch 3/6 - Validation: 100%|██████████████████████████████████████████| 228/228 [00:07<00:00, 30.34it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Epoch 3/6 - Training Loss: 0.5532, Training Accuracy: 0.7215, Validation Accuracy: 0.6208\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Epoch 4/6 - Training:   0%|                                                      | 0/683 [00:00<?, ?it/s]/var/folders/hs/br_4rpdj68nc3sfdpgv0xgn80000gn/T/ipykernel_12767/33828928.py:18: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
-      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
-      "Epoch 4/6 - Training: 100%|████████████████████████████████████████████| 683/683 [01:22<00:00,  8.24it/s]\n",
-      "Epoch 4/6 - Validation:   0%|                                                    | 0/228 [00:00<?, ?it/s]/var/folders/hs/br_4rpdj68nc3sfdpgv0xgn80000gn/T/ipykernel_12767/33828928.py:18: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
-      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
-      "Epoch 4/6 - Validation: 100%|██████████████████████████████████████████| 228/228 [00:07<00:00, 30.29it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Epoch 4/6 - Training Loss: 0.4508, Training Accuracy: 0.7918, Validation Accuracy: 0.6419\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Epoch 5/6 - Training:   0%|                                                      | 0/683 [00:00<?, ?it/s]/var/folders/hs/br_4rpdj68nc3sfdpgv0xgn80000gn/T/ipykernel_12767/33828928.py:18: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
-      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
-      "Epoch 5/6 - Training: 100%|████████████████████████████████████████████| 683/683 [01:22<00:00,  8.27it/s]\n",
-      "Epoch 5/6 - Validation:   0%|                                                    | 0/228 [00:00<?, ?it/s]/var/folders/hs/br_4rpdj68nc3sfdpgv0xgn80000gn/T/ipykernel_12767/33828928.py:18: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
-      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
-      "Epoch 5/6 - Validation: 100%|██████████████████████████████████████████| 228/228 [00:07<00:00, 29.08it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Epoch 5/6 - Training Loss: 0.3443, Training Accuracy: 0.8579, Validation Accuracy: 0.6371\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Epoch 6/6 - Training:   0%|                                                      | 0/683 [00:00<?, ?it/s]/var/folders/hs/br_4rpdj68nc3sfdpgv0xgn80000gn/T/ipykernel_12767/33828928.py:18: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
-      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
-      "Epoch 6/6 - Training: 100%|████████████████████████████████████████████| 683/683 [01:22<00:00,  8.26it/s]\n",
-      "Epoch 6/6 - Validation:   0%|                                                    | 0/228 [00:00<?, ?it/s]/var/folders/hs/br_4rpdj68nc3sfdpgv0xgn80000gn/T/ipykernel_12767/33828928.py:18: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
-      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
-      "Epoch 6/6 - Validation: 100%|██████████████████████████████████████████| 228/228 [00:07<00:00, 29.62it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Epoch 6/6 - Training Loss: 0.2608, Training Accuracy: 0.8963, Validation Accuracy: 0.6322\n",
-      "Training completed.\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "seed_value = 42\n",
     "\n",