trainCoherenceModelNews.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import pandas as pd\n",
    "from coherenceModel import *\n",
    "from torch.utils.data import DataLoader\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paragraph</th>\n",
       "      <th>is_coherent</th>\n",
       "      <th>id</th>\n",
       "      <th>num_sent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A new supercomputer model of the universe...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1. ``Mrs. Doubtfire''     2. ``A Perfect ...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Five key U.S. maritime unions formally ag...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OUR HOURLY BREAD: Bread doesn't demand mu...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LYON, France  &amp;MD  Much of the criticism ...</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41995</th>\n",
       "      <td>Before he can leave the United States, Abu Mar...</td>\n",
       "      <td>0</td>\n",
       "      <td>1999</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41996</th>\n",
       "      <td>State Department spokesman Nicholas Burns said...</td>\n",
       "      <td>0</td>\n",
       "      <td>1999</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41997</th>\n",
       "      <td>``The decision was taken on the basis of overa...</td>\n",
       "      <td>0</td>\n",
       "      <td>1999</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41998</th>\n",
       "      <td>An INS spokesman said the agency's case to exc...</td>\n",
       "      <td>0</td>\n",
       "      <td>1999</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41999</th>\n",
       "      <td>An INS spokesman said the agency's case to exc...</td>\n",
       "      <td>0</td>\n",
       "      <td>1999</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>42000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               paragraph  is_coherent    id  \\\n",
       "0           A new supercomputer model of the universe...            1     0   \n",
       "1           1. ``Mrs. Doubtfire''     2. ``A Perfect ...            1     1   \n",
       "2           Five key U.S. maritime unions formally ag...            1     2   \n",
       "3           OUR HOURLY BREAD: Bread doesn't demand mu...            1     3   \n",
       "4           LYON, France  &MD  Much of the criticism ...            1     4   \n",
       "...                                                  ...          ...   ...   \n",
       "41995  Before he can leave the United States, Abu Mar...            0  1999   \n",
       "41996  State Department spokesman Nicholas Burns said...            0  1999   \n",
       "41997  ``The decision was taken on the basis of overa...            0  1999   \n",
       "41998  An INS spokesman said the agency's case to exc...            0  1999   \n",
       "41999  An INS spokesman said the agency's case to exc...            0  1999   \n",
       "\n",
       "       num_sent  \n",
       "0            12  \n",
       "1            11  \n",
       "2            12  \n",
       "3            12  \n",
       "4            12  \n",
       "...         ...  \n",
       "41995        11  \n",
       "41996        11  \n",
       "41997        11  \n",
       "41998        11  \n",
       "41999        11  \n",
       "\n",
       "[42000 rows x 4 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paragraph_df = pd.read_csv('LATimesWashPostPerms.csv')\n",
    "paragraph_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.047619047619047616\n",
      "0.047619047619047616\n",
      "0.047619047619047616\n",
      "     A new supercomputer model of the universe  &MD  the most complex such simulation ever created  &MD  lends credence to the theory that the original recipe for the real cosmos probably included both cold and hot versions of the mysterious ingredient known as dark matter. An astrophysics team at the University of Illinois used a massively parallel supercomputer to visualize the universe as it would be seen in X-rays, which are emitted by superhot gases or in violent events. ``Our simulation is the first that is sufficiently comprehensive to make theoretical predictions that can be compared with observations,'' said team leader Michael Norman, who presented the work at a meeting of the American Astronomical Society last week in Minneapolis. The new model agrees so well with actual observations, he said, that it may guide astronomers toward new discoveries. In the last decade, many astronomers came to believe that at least 98 percent of the universe consists of dark matter: invisible material of an unknown nature, detected only through the effects of its gravity. But observations soon eliminated notions that all this matter might be hot  &MD  that is, having high energy. Eventually, a New Mexico State University model showed that a mix of cold (low-energy) and hot could explain the observed state of the cosmos. However, that model left out ordinary matter. Norman's model is more complex, computing how (in one theory) cosmic gas would be heated as it was pulled into the gravitational ``wells'' created by dark matter at almost 190 miles per second, creating shock waves that superheat intergalactic gases. The results showed X-ray-emitting gas clusters everywhere in a cosmos closely resembling recent X-ray observations by ROSAT, an orbiting NASA-European satellite. The simulation, funded by NASA and the National Science Foundation, was modeled as a cube divided into 512 cells along each edge, with each cell representing a volume of space 1 million light-years on a side. The Connection Machine-5 supercomputer took 30 hours to build this universe in a box.\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "train_pars = paragraph_df[paragraph_df['id'] < 1440]\n",
    "dev_pars = paragraph_df[(paragraph_df['id'] >= 1440) & (paragraph_df['id'] < 1800)]\n",
    "test_pars = paragraph_df[paragraph_df['id'] >= 1800]\n",
    "\n",
    "def get_balance(df):\n",
    "    print(len(df[df['is_coherent'] == 1]) / len(df))\n",
    "\n",
    "X_train, y_train = train_pars.paragraph.values, train_pars.is_coherent.values\n",
    "X_val, y_val = dev_pars.paragraph.values, dev_pars.is_coherent.values\n",
    "X_test, y_test = test_pars.paragraph.values, test_pars.is_coherent.values\n",
    "\n",
    "get_balance(train_pars)\n",
    "get_balance(dev_pars)\n",
    "get_balance(test_pars)\n",
    "\n",
    "print(X_train[0])\n",
    "print(y_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim.downloader\n",
    "embed = gensim.downloader.load(\"glove-wiki-gigaword-50\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Complete\n",
      "Complete\n",
      "Complete\n"
     ]
    }
   ],
   "source": [
    "################ SET THIS TO CHANGE WINDOW SIZE OF THINGS BELOW:\n",
    "wsize = 5\n",
    "################\n",
    "train_data = WindowedParDataset(X_train, y_train, embed, wsize)\n",
    "dev_data = WindowedParDataset(X_val, y_val, embed, wsize)\n",
    "test_data = WindowedParDataset(X_test, y_test, embed, wsize)\n",
    "train_loader = DataLoader(train_data, batch_size=25, collate_fn=basic_collate_fn, shuffle=True)\n",
    "dev_loader = DataLoader(dev_data, batch_size=25, collate_fn=basic_collate_fn, shuffle=True)\n",
    "test_loader = DataLoader(test_data, batch_size=25, collate_fn=basic_collate_fn, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([20.], device='cuda:0')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_pos = len(paragraph_df[paragraph_df[\"is_coherent\"] == 1])\n",
    "num_neg = len(paragraph_df[paragraph_df[\"is_coherent\"] == 0])\n",
    "dampen = 1\n",
    "pos_weight = torch.Tensor([num_neg / num_pos / dampen]).to(device)\n",
    "pos_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------ Start Training ------------------------\n",
      "Epoch No. 1--Iteration No. 2181-- batch loss = 0.6201\n",
      "Validation UAR: 0.5384\n",
      "Validation accuracy: 0.8352\n",
      "Validation loss: 1.3365\n",
      "Epoch No. 2--Iteration No. 4362-- batch loss = 1.2954\n",
      "Validation UAR: 0.5574\n",
      "Validation accuracy: 0.4538\n",
      "Validation loss: 1.3217\n",
      "Epoch No. 3--Iteration No. 6543-- batch loss = 1.8060\n",
      "Validation UAR: 0.5663\n",
      "Validation accuracy: 0.4141\n",
      "Validation loss: 1.3144\n",
      "Epoch No. 4--Iteration No. 8724-- batch loss = 0.9277\n",
      "Validation UAR: 0.5802\n",
      "Validation accuracy: 0.6283\n",
      "Validation loss: 1.3045\n",
      "Epoch No. 5--Iteration No. 10905-- batch loss = 2.2346\n",
      "Validation UAR: 0.5878\n",
      "Validation accuracy: 0.5773\n",
      "Validation loss: 1.2957\n",
      "Epoch No. 6--Iteration No. 13086-- batch loss = 2.1687\n",
      "Validation UAR: 0.5942\n",
      "Validation accuracy: 0.5421\n",
      "Validation loss: 1.2874\n",
      "Epoch No. 7--Iteration No. 15267-- batch loss = 0.7313\n",
      "Validation UAR: 0.5982\n",
      "Validation accuracy: 0.4892\n",
      "Validation loss: 1.2803\n",
      "Epoch No. 8--Iteration No. 17448-- batch loss = 1.8343\n",
      "Validation UAR: 0.6059\n",
      "Validation accuracy: 0.6174\n",
      "Validation loss: 1.2711\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[10], line 5\u001b[0m\n\u001b[1;32m      3\u001b[0m ffnn\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[1;32m      4\u001b[0m optim \u001b[38;5;241m=\u001b[39m get_optimizer(ffnn, lr\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1e-2\u001b[39m, weight_decay\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m)\n\u001b[0;32m----> 5\u001b[0m best_model, stats \u001b[38;5;241m=\u001b[39m \u001b[43mtrain_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43mffnn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdev_loader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdev_loader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moptim\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpos_weight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpos_weight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      6\u001b[0m \u001b[43m                                \u001b[49m\u001b[43mnum_epoch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m15\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcollect_cycle\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m20\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpatience\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m      7\u001b[0m plot_loss(stats)\n",
      "File \u001b[0;32m~/class/487/project/coherenceModel.py:191\u001b[0m, in \u001b[0;36mtrain_model\u001b[0;34m(net, trn_loader, val_loader, optim, pos_weight, num_epoch, collect_cycle, device, verbose, patience, stopping_criteria)\u001b[0m\n\u001b[1;32m    189\u001b[0m             output \u001b[38;5;241m=\u001b[39m net(windows)\n\u001b[1;32m    190\u001b[0m             loss \u001b[38;5;241m=\u001b[39m calculate_loss(output, labels, loss_fn)\n\u001b[0;32m--> 191\u001b[0m             \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    192\u001b[0m             optim\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m    194\u001b[0m \u001b[38;5;66;03m#             print_grads(net)\u001b[39;00m\n",
      "File \u001b[0;32m~/class/448/eecs448/lib/python3.8/site-packages/torch/_tensor.py:487\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    477\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    478\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m    479\u001b[0m         Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m    480\u001b[0m         (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    485\u001b[0m         inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m    486\u001b[0m     )\n\u001b[0;32m--> 487\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    488\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\n\u001b[1;32m    489\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/class/448/eecs448/lib/python3.8/site-packages/torch/autograd/__init__.py:200\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    195\u001b[0m     retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m    197\u001b[0m \u001b[38;5;66;03m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[1;32m    198\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m    199\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 200\u001b[0m \u001b[43mVariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execution_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m    201\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    202\u001b[0m \u001b[43m    \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# test on validation to see if overfit is possible\n",
    "ffnn = FFNN(5, device)\n",
    "ffnn.to(device)\n",
    "optim = get_optimizer(ffnn, lr=1e-2, weight_decay=0)\n",
    "best_model, stats = train_model(ffnn, dev_loader, dev_loader, optim, pos_weight=pos_weight,\n",
    "                                num_epoch=15, collect_cycle=20, device=device, patience=None)\n",
    "plot_loss(stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "learning rate from: [0.01]\n",
      "weight_decay from: [0.0002, 0.002, 0.005, 0.01]\n",
      "window from: [5]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "338a71c05b9240c18d46173c869a6484",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------ Start Training ------------------------\n",
      "Epoch No. 1--Iteration No. 8646-- batch loss = 2.0668\n",
      "Validation UAR: 0.5278\n",
      "Validation accuracy: 0.3419\n",
      "Validation loss: 1.3305\n",
      "Epoch No. 2--Iteration No. 17292-- batch loss = 1.3664\n",
      "Validation UAR: 0.6198\n",
      "Validation accuracy: 0.3861\n",
      "Validation loss: 1.2688\n",
      "Epoch No. 3--Iteration No. 25938-- batch loss = 0.6299\n",
      "Validation UAR: 0.6084\n",
      "Validation accuracy: 0.4007\n",
      "Validation loss: 1.2038\n",
      "Epoch No. 4--Iteration No. 34584-- batch loss = 0.6006\n",
      "Validation UAR: 0.6192\n",
      "Validation accuracy: 0.4590\n",
      "Validation loss: 1.2020\n",
      "Epoch No. 5--Iteration No. 43230-- batch loss = 1.2345\n",
      "Validation UAR: 0.6220\n",
      "Validation accuracy: 0.3635\n",
      "Validation loss: 1.1849\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "torch.cuda.empty_cache()\n",
    "\n",
    "def search_param_utterance(wsize):\n",
    "    \"\"\"Experiemnt on different hyper parameters.\"\"\"\n",
    "    learning_rate, weight_decay = get_hyper_parameters()\n",
    "    window_sizes = [wsize]\n",
    "    print(\"learning rate from: {}\\nweight_decay from: {}\\nwindow from: {}\".format(\n",
    "        learning_rate, weight_decay, window_sizes\n",
    "    ))\n",
    "    best_model, best_stats = None, None\n",
    "    best_accuracy, best_lr, best_wd, best_window_size = 0, 0, 0, 0\n",
    "    for lr, wd, window_size in tqdm(itertools.product(learning_rate, weight_decay, window_sizes),\n",
    "                           total=len(learning_rate) * len(weight_decay) * len(window_sizes)):\n",
    "        net = FFNN(window_size, device).to(device)\n",
    "        optim = get_optimizer(net, lr=lr, weight_decay=wd)\n",
    "        model, stats = train_model(net, train_loader, dev_loader, optim, pos_weight=pos_weight, \n",
    "                                   num_epoch=100, collect_cycle=500, device=device, \n",
    "                                   verbose=True, patience=5, stopping_criteria='accuracy')\n",
    "        # print accuracy\n",
    "        print(f\"{(lr, wd, window_size)}: {stats['accuracy']}\")\n",
    "        # update best parameters if needed\n",
    "        if stats['accuracy'] > best_accuracy:\n",
    "            best_accuracy = stats['accuracy']\n",
    "            best_model, best_stats = model, stats\n",
    "            best_lr, best_wd, best_window_size = lr, wd, window_size\n",
    "            torch.save(best_model.state_dict(), 'best_rnn_latimes.pt')\n",
    "    print(\"\\n\\nBest learning rate: {}, best weight_decay: {}, best window: {}\".format(\n",
    "        best_lr, best_wd, best_window_size))\n",
    "    print(\"Accuracy: {:.4f}\".format(best_accuracy))\n",
    "    plot_loss(best_stats)\n",
    "    return best_model\n",
    "basic_model = search_param_utterance(wsize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final selection: window size 5 with Q = 0.0002\n",
      "Test UAR: 0.7660\n",
      "Test accuracy: 0.7768\n",
      "Test loss: 0.9321\n"
     ]
    }
   ],
   "source": [
    "basic_model = FFNN(5, device)\n",
    "basic_model.load_state_dict(torch.load('best_rnn.pt'))\n",
    "basic_model.eval()\n",
    "basic_model.to(device)\n",
    "uar, accuracy, total_loss = get_validation_performance(\n",
    "    basic_model, \n",
    "    nn.BCEWithLogitsLoss(pos_weight=pos_weight), \n",
    "    test_loader, \n",
    "    device\n",
    ")\n",
    "print(\"Final selection: window size 5 with Q = 0.0002\")\n",
    "print(\"Test UAR: {:.4f}\".format(uar))\n",
    "print(\"Test accuracy: {:.4f}\".format(accuracy))\n",
    "print(\"Test loss: {:.4f}\".format(total_loss))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}