WeeBit_CNN_LSTM_Attention.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "language": "python",
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "version": "3.6.4",
      "file_extension": ".py",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "name": "python",
      "mimetype": "text/x-python"
    },
    "colab": {
      "name": "WeeBit_CNN_LSTM_Attention.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "I2mjxZHn8JF6",
        "0fxt1r0eIKHB"
      ],
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/m607stars/Reading-WeeBit-with-LSTMs-CNNs/blob/main/WeeBit_CNN_LSTM_Attention.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NVC6j9XOC0J1"
      },
      "source": [
        "# Imports and Installs\n",
        "\n",
        "Note: DO NOT forget to change the path for saving models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Upcb-2aDVp3I",
        "outputId": "53912048-5e37-4f29-cf26-a31ae5be9ae0"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-17T13:04:20.192187Z",
          "iopub.execute_input": "2021-06-17T13:04:20.192588Z",
          "iopub.status.idle": "2021-06-17T13:04:20.196832Z",
          "shell.execute_reply.started": "2021-06-17T13:04:20.192493Z",
          "shell.execute_reply": "2021-06-17T13:04:20.195993Z"
        },
        "trusted": true,
        "id": "4DDm-bAEC0KA"
      },
      "source": [
        "# !pip install torchviz"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-17T13:04:31.917757Z",
          "iopub.execute_input": "2021-06-17T13:04:31.91814Z",
          "iopub.status.idle": "2021-06-17T13:04:34.255424Z",
          "shell.execute_reply.started": "2021-06-17T13:04:31.918108Z",
          "shell.execute_reply": "2021-06-17T13:04:34.25454Z"
        },
        "trusted": true,
        "id": "R7_VU3rsC0KD"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import os\n",
        "import torch\n",
        "from nltk.tokenize import word_tokenize\n",
        "from torchtext import vocab\n",
        "import random\n",
        "import nltk\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import classification_report"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9wCKUBIfaWyn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6e56f9ad-682a-4422-c6d5-b0a037dd0ce7"
      },
      "source": [
        "nltk.download('punkt')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aJNC91OVSJow",
        "outputId": "11d2b4b4-d919-4d9f-b5b7-fe386fa2dafb"
      },
      "source": [
        "def seed_everything(seed):\n",
        "    random.seed(seed)\n",
        "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "    np.random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    torch.cuda.manual_seed(seed)\n",
        "    print(\"Seeding done\")\n",
        "seed_everything(42)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Seeding done\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-17T13:04:34.257209Z",
          "iopub.execute_input": "2021-06-17T13:04:34.257534Z",
          "iopub.status.idle": "2021-06-17T13:05:23.83085Z",
          "shell.execute_reply.started": "2021-06-17T13:04:34.257499Z",
          "shell.execute_reply": "2021-06-17T13:05:23.829972Z"
        },
        "trusted": true,
        "id": "HVrapbC9C0KE"
      },
      "source": [
        "VECTOR_PATH = '/content/drive/MyDrive/Readability_Research_Paper/'\n",
        "VECTOR_NAME = 'glove.6B.300d.txt'\n",
        "\n",
        "TEXT_LENGTH = 187\n",
        "EMBEDDING_SIZE = 300\n",
        "HIDDEN_SIZE = 200\n",
        "BATCH_SIZE=16\n",
        "\n",
        "embeddings = vocab.Vectors(VECTOR_NAME,VECTOR_PATH)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-17T13:05:23.832382Z",
          "iopub.execute_input": "2021-06-17T13:05:23.832712Z",
          "iopub.status.idle": "2021-06-17T13:05:23.913894Z",
          "shell.execute_reply.started": "2021-06-17T13:05:23.832671Z",
          "shell.execute_reply": "2021-06-17T13:05:23.912978Z"
        },
        "trusted": true,
        "id": "n-hproQLC0KF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dc1ea300-22a1-4f88-ee96-2c047eda3d76"
      },
      "source": [
        "if torch.cuda.is_available():\n",
        "    device = torch.device('cuda')\n",
        "else:\n",
        "    device = torch.device('cpu')\n",
        "    \n",
        "print(device)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-17T13:05:23.915185Z",
          "iopub.execute_input": "2021-06-17T13:05:23.915509Z",
          "iopub.status.idle": "2021-06-17T13:05:24.025958Z",
          "shell.execute_reply.started": "2021-06-17T13:05:23.915472Z",
          "shell.execute_reply": "2021-06-17T13:05:24.025147Z"
        },
        "trusted": true,
        "id": "6f-ITJlLC0KG",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "f5607742-7af4-4686-e9fa-b4c9c2d3d00a"
      },
      "source": [
        "train_dataset = pd.read_csv('/content/drive/MyDrive/Readability_Research_Paper/train.csv')\n",
        "train_dataset['readability'] = train_dataset['readability'].apply(lambda x: x-2)\n",
        "train_dataset.head()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>readability</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>they sent me a salwar kameezpeacockblueand ano...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>the chart shows each planet and its number of ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>this revision bite will help you understand wh...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>what are powers and roots find out how they work</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>the wright brothers flew the first airplane ne...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  readability\n",
              "0  they sent me a salwar kameezpeacockblueand ano...            4\n",
              "1  the chart shows each planet and its number of ...            0\n",
              "2  this revision bite will help you understand wh...            4\n",
              "3   what are powers and roots find out how they work            3\n",
              "4  the wright brothers flew the first airplane ne...            0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-17T13:05:24.027208Z",
          "iopub.execute_input": "2021-06-17T13:05:24.027531Z",
          "iopub.status.idle": "2021-06-17T13:05:24.047821Z",
          "shell.execute_reply.started": "2021-06-17T13:05:24.027495Z",
          "shell.execute_reply": "2021-06-17T13:05:24.047052Z"
        },
        "trusted": true,
        "id": "jPeUbL3eC0KG",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "7b269c9f-3018-46ca-e3ac-498a1ebd2d42"
      },
      "source": [
        "test_dataset = pd.read_csv('/content/drive/MyDrive/Readability_Research_Paper/test.csv')\n",
        "test_dataset['readability'] = test_dataset['readability'].apply(lambda x: x-2)\n",
        "test_dataset.head()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>readability</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>to understand a work of art or a beautiful obj...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>perhaps the most important of these is the use...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>q what is a tornados favorite game a twister</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>the final thing to remember is there are lots ...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3d shapes have 3dimensions length width and de...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  readability\n",
              "0  to understand a work of art or a beautiful obj...            4\n",
              "1  perhaps the most important of these is the use...            4\n",
              "2       q what is a tornados favorite game a twister            0\n",
              "3  the final thing to remember is there are lots ...            4\n",
              "4  3d shapes have 3dimensions length width and de...            3"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "27-DvDgAC0KH"
      },
      "source": [
        "# Data pre-processing and DataLoaders"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-17T13:05:25.130689Z",
          "iopub.execute_input": "2021-06-17T13:05:25.131097Z",
          "iopub.status.idle": "2021-06-17T13:05:25.236336Z",
          "shell.execute_reply.started": "2021-06-17T13:05:25.131052Z",
          "shell.execute_reply": "2021-06-17T13:05:25.235398Z"
        },
        "trusted": true,
        "id": "_JAJiGSIC0KJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fc6be356-04a5-4afb-c5ea-82e89ab58163"
      },
      "source": [
        "def get_word_to_index(texts):\n",
        "    word_to_index = {\n",
        "        '<PAD>':0,\n",
        "        '<START>':1,\n",
        "        '<END>':2,\n",
        "    }\n",
        "    ind = 3\n",
        "    for text in texts:\n",
        "        words = word_tokenize(text)\n",
        "        for word in words:\n",
        "            if word not in word_to_index.keys():\n",
        "                word_to_index[word] = ind\n",
        "                ind += 1\n",
        "                \n",
        "    return word_to_index   \n",
        "\n",
        "word_to_index_dict = get_word_to_index(train_dataset['text'])\n",
        "VOCABULARY_SIZE = len(word_to_index_dict.keys())\n",
        "print(VOCABULARY_SIZE)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "21072\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-17T13:06:02.472213Z",
          "iopub.execute_input": "2021-06-17T13:06:02.472542Z",
          "iopub.status.idle": "2021-06-17T13:06:02.49165Z",
          "shell.execute_reply.started": "2021-06-17T13:06:02.472511Z",
          "shell.execute_reply": "2021-06-17T13:06:02.49062Z"
        },
        "trusted": true,
        "id": "33P8dmcMC0KJ"
      },
      "source": [
        "def get_tensor_from_text(text):\n",
        "    word_list = []\n",
        "    words = word_tokenize(text)\n",
        "    for word in words:\n",
        "        word_list.append(word_to_index_dict[word])\n",
        "    if len(word_list) > TEXT_LENGTH:\n",
        "        word_list = word_list[:TEXT_LENGTH]\n",
        "    else:\n",
        "        word_list.extend([0]*(TEXT_LENGTH-len(word_list)))\n",
        "    \n",
        "    tensor_list = torch.tensor(word_list, device=device, dtype=torch.long)\n",
        "    return tensor_list"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-17T13:06:04.311629Z",
          "iopub.execute_input": "2021-06-17T13:06:04.311985Z",
          "iopub.status.idle": "2021-06-17T13:06:04.317795Z",
          "shell.execute_reply.started": "2021-06-17T13:06:04.311954Z",
          "shell.execute_reply": "2021-06-17T13:06:04.31686Z"
        },
        "trusted": true,
        "id": "kL2XMh6hC0KK"
      },
      "source": [
        "class WeebitDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self,dataset):\n",
        "        self.dataset = dataset\n",
        "    \n",
        "    def __getitem__(self,index): \n",
        "        text = self.dataset['text'].iloc[index]\n",
        "        y = torch.tensor(self.dataset['readability'].iloc[index],dtype=torch.float,device=device)\n",
        "        x = get_tensor_from_text(text)\n",
        "        return x,y\n",
        "    \n",
        "    def __len__(self):\n",
        "        return len(self.dataset)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7M_DsPplMbRI"
      },
      "source": [
        "def create_embedding_matrix(embeddings,vocabulary_size):  \n",
        "    embedding_matrix = np.random.rand(vocabulary_size,EMBEDDING_SIZE)\n",
        "    for string,index in word_to_index_dict.items():\n",
        "        if not  all(x == 0 for x in embeddings[string].tolist()):\n",
        "            embedding_matrix[index] = embeddings[string] \n",
        "    return embedding_matrix"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Q25Fe9HYumo"
      },
      "source": [
        "embedding_matrix = create_embedding_matrix(embeddings,VOCABULARY_SIZE)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i2y2FVnQ_Taa",
        "outputId": "0dd40d37-a451-41c0-a3e3-9e5aef53f104"
      },
      "source": [
        "print(embedding_matrix.shape)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(21072, 300)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kcEGq9VHC0KL"
      },
      "source": [
        "# Model and Training "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I2mjxZHn8JF6"
      },
      "source": [
        "## Done"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ktrWzCQLHjjV"
      },
      "source": [
        "# Mayank\n",
        "\n",
        "class ReadabilityModel_LSTM(torch.nn.Module):\n",
        "    def __init__(self, embedding_matrix, hidden_size, num_of_layers):\n",
        "        super().__init__()\n",
        "        self.dropout_probability = 0.2\n",
        "        self.embeddings_layer = torch.nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n",
        "                                                   padding_idx=0)\n",
        "        self.embeddings_layer.weight = torch.nn.Parameter(torch.tensor(embedding_matrix, \n",
        "                                                                       dtype=torch.float32,\n",
        "                                                                       device=device))\n",
        "        self.embeddings_layer.weight.requires_grad = False\n",
        "        self.lstm_layer = torch.nn.LSTM(300, hidden_size,\n",
        "                                        num_layers=num_of_layers, dropout=self.dropout_probability,\n",
        "                                        batch_first=True, bidirectional=True)\n",
        "        self.linear_layer_1 = torch.nn.Linear(hidden_size,128)\n",
        "        self.dropout_layer = torch.nn.Dropout(p=self.dropout_probability)\n",
        "        self.leaky_relu = torch.nn.LeakyReLU()\n",
        "        self.linear_layer_2 = torch.nn.Linear(128,5)\n",
        "\n",
        "    def forward(self,x):\n",
        "        x = self.embeddings_layer(x) #shape of x is 16,187,300\n",
        "        lstm_output, (ht, ct) = self.lstm_layer(x)\n",
        "        #Shape of lstm_output_1 is 16,50,512 \n",
        "        #Shape of ht1 is 6,16,256 \n",
        "        #Shape of ct2 is 6,16,256\n",
        "        linear_output_1 = self.linear_layer_1(ht[-1])\n",
        "        linear_output_1 = self.leaky_relu(linear_output_1)\n",
        "        linear_output_1 = self.dropout_layer(linear_output_1)\n",
        "        linear_output_2 = self.linear_layer_2(linear_output_1)\n",
        "        output = linear_output_2\n",
        "        return output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-17T08:42:03.11576Z",
          "iopub.status.idle": "2021-06-17T08:42:03.116141Z"
        },
        "trusted": true,
        "id": "O-EhfRx3C0KL"
      },
      "source": [
        "# Mayank\n",
        "\n",
        "class ReadabilityModel_1_CNN_LSTM(torch.nn.Module):\n",
        "    def __init__(self, embedding_matrix, hidden_size, num_of_layers):\n",
        "        super().__init__()\n",
        "        self.dropout_probability = 0.2\n",
        "        self.embeddings_layer = torch.nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n",
        "                                                   padding_idx=0)\n",
        "        self.embeddings_layer.weight = torch.nn.Parameter(torch.tensor(embedding_matrix, \n",
        "                                                                       dtype=torch.float32,\n",
        "                                                                       device=device))\n",
        "        self.embeddings_layer.weight.requires_grad = False\n",
        "        self.conv_layer_1 = torch.nn.Conv1d(in_channels=300,out_channels=300,kernel_size=3,stride=1)\n",
        "        self.lstm_layer = torch.nn.LSTM(300, hidden_size,\n",
        "                                        num_layers=num_of_layers, dropout=self.dropout_probability,\n",
        "                                        batch_first=True, bidirectional=True)\n",
        "        self.linear_layer_1 = torch.nn.Linear(hidden_size,128)\n",
        "        self.linear_layer_2 = torch.nn.Linear(128,64)\n",
        "        self.linear_layer_3 = torch.nn.Linear(64,16)\n",
        "        self.linear_layer_4 = torch.nn.Linear(16,5)\n",
        "        self.dropout_layer = torch.nn.Dropout(p=self.dropout_probability)\n",
        "        self.leaky_relu = torch.nn.LeakyReLU() \n",
        "\n",
        "    def forward(self,x):\n",
        "        x = self.embeddings_layer(x) #shape of x is 16,187,300\n",
        "        x = x.permute(0,2,1) #shape of x is 16,300,187\n",
        "        conv_output_1 = self.conv_layer_1(x) #shape of conv_output_1 is 16,300,185\n",
        "        conv_output = conv_output_1.permute(0,2,1)  #shape of conv_output_2 is 16,61,300\n",
        "        lstm_output, (ht, ct) = self.lstm_layer(conv_output)\n",
        "        #Shape of lstm_output_1 is 16,300,512 \n",
        "        #Shape of ht1 is 6,16,256 \n",
        "        #Shape of ct2 is 6,16,256\n",
        "        linear_output_1 = self.linear_layer_1(ht[-1])\n",
        "        linear_output_1 = self.leaky_relu(linear_output_1)\n",
        "        linear_output_1 = self.dropout_layer(linear_output_1)\n",
        "\n",
        "        linear_output_2 = self.linear_layer_2(linear_output_1)\n",
        "        linear_output_2 = self.leaky_relu(linear_output_2)\n",
        "        linear_output_2 = self.dropout_layer(linear_output_2)\n",
        "\n",
        "        linear_output_3 = self.linear_layer_3(linear_output_2)\n",
        "        linear_output_3 = self.leaky_relu(linear_output_3)\n",
        "        linear_output_3 = self.dropout_layer(linear_output_3)\n",
        "\n",
        "        linear_output_4 = self.linear_layer_4(linear_output_3)\n",
        "        output = linear_output_4\n",
        "        return output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fDsoeBUpkPyM"
      },
      "source": [
        "# Mayank\n",
        "\n",
        "class ReadabilityModel_2_CNN_LSTM(torch.nn.Module):\n",
        "    def __init__(self, embedding_matrix, hidden_size, num_of_layers):\n",
        "        super().__init__()\n",
        "        self.dropout_probability = 0.2\n",
        "        self.embeddings_layer = torch.nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n",
        "                                                   padding_idx=0)\n",
        "        self.embeddings_layer.weight = torch.nn.Parameter(torch.tensor(embedding_matrix, \n",
        "                                                                       dtype=torch.float32,\n",
        "                                                                       device=device))\n",
        "        self.embeddings_layer.weight.requires_grad = False\n",
        "        self.conv_layer_1 = torch.nn.Conv1d(in_channels=300,out_channels=300,kernel_size=3,stride=1)\n",
        "        self.conv_layer_2 = torch.nn.Conv1d(in_channels=300,out_channels=300,kernel_size=5,stride=3)\n",
        "        self.lstm_layer = torch.nn.LSTM(300, hidden_size,\n",
        "                                        num_layers=num_of_layers, dropout=self.dropout_probability,\n",
        "                                        batch_first=True, bidirectional=True)\n",
        "        self.linear_layer_1 = torch.nn.Linear(hidden_size,128)\n",
        "        self.linear_layer_2 = torch.nn.Linear(128,64)\n",
        "        self.linear_layer_3 = torch.nn.Linear(64,16)\n",
        "        self.linear_layer_4 = torch.nn.Linear(16,5)\n",
        "        self.dropout_layer = torch.nn.Dropout(p=self.dropout_probability)\n",
        "        self.leaky_relu = torch.nn.LeakyReLU() \n",
        "\n",
        "    def forward(self,x):\n",
        "        x = self.embeddings_layer(x) #shape of x is 16,187,300\n",
        "        x = x.permute(0,2,1) #shape of x is 16,300,187\n",
        "        conv_output_1 = self.conv_layer_1(x) #shape of conv_output_1 is 16,300,185\n",
        "        conv_output_2 = self.conv_layer_2(conv_output_1) #shape of conv_output_2 is 16,300,61\n",
        "        conv_output = conv_output_1.permute(0,2,1)  #shape of conv_output_2 is 16,61,300\n",
        "        lstm_output, (ht, ct) = self.lstm_layer(conv_output)\n",
        "        #Shape of lstm_output_1 is 16,300,512 \n",
        "        #Shape of ht1 is 6,16,256 \n",
        "        #Shape of ct2 is 6,16,256\n",
        "        linear_output_1 = self.linear_layer_1(ht[-1])\n",
        "        linear_output_1 = self.leaky_relu(linear_output_1)\n",
        "        linear_output_1 = self.dropout_layer(linear_output_1)\n",
        "\n",
        "        linear_output_2 = self.linear_layer_2(linear_output_1)\n",
        "        linear_output_2 = self.leaky_relu(linear_output_2)\n",
        "        linear_output_2 = self.dropout_layer(linear_output_2)\n",
        "\n",
        "        linear_output_3 = self.linear_layer_3(linear_output_2)\n",
        "        linear_output_3 = self.leaky_relu(linear_output_3)\n",
        "        linear_output_3 = self.dropout_layer(linear_output_3)\n",
        "\n",
        "        linear_output_4 = self.linear_layer_4(linear_output_3)\n",
        "        output = linear_output_4\n",
        "        return output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mY3FXy3RmTkL"
      },
      "source": [
        "# Mayank\n",
        "\n",
        "class ReadabilityModel_3_CNNs(torch.nn.Module):\n",
        "    def __init__(self, embedding_matrix, hidden_size, num_of_layers):\n",
        "        super().__init__()\n",
        "        self.dropout_probability = 0.2\n",
        "        self.embeddings_layer = torch.nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n",
        "                                                   padding_idx=0)\n",
        "        self.embeddings_layer.weight = torch.nn.Parameter(torch.tensor(embedding_matrix, \n",
        "                                                                       dtype=torch.float32,\n",
        "                                                                       device=device))\n",
        "        self.embeddings_layer.weight.requires_grad = False\n",
        "        self.conv_layer_1 = torch.nn.Conv1d(in_channels=300,out_channels=300,kernel_size=3,stride=1)\n",
        "        self.conv_layer_2 = torch.nn.Conv1d(in_channels=300,out_channels=300,kernel_size=5,stride=3)\n",
        "        self.conv_layer_3 = torch.nn.Conv1d(in_channels=300,out_channels=300,kernel_size=7,stride=9)\n",
        "        self.conv_linear_layer_1 = torch.nn.Linear(185,128)\n",
        "        self.conv_linear_layer_2 = torch.nn.Linear(61,128)\n",
        "        self.conv_linear_layer_3 = torch.nn.Linear(21,128)\n",
        "\n",
        "        self.linear_layer_1 = torch.nn.Linear(900,256)\n",
        "        self.linear_layer_2 = torch.nn.Linear(256,64)\n",
        "        self.linear_layer_3 = torch.nn.Linear(64,8)\n",
        "        self.linear_layer_4 = torch.nn.Linear(8,1)\n",
        "        self.linear_layer_5 = torch.nn.Linear(128,32)\n",
        "        self.linear_layer_6 = torch.nn.Linear(32,5)\n",
        "        self.dropout_layer = torch.nn.Dropout(p=self.dropout_probability)\n",
        "        self.leaky_relu = torch.nn.LeakyReLU() \n",
        "\n",
        "    def forward(self,x):\n",
        "        x = self.embeddings_layer(x) #shape of x is 16,187,300\n",
        "        x = x.permute(0,2,1) #shape of x is 16,300,187\n",
        "        conv_output_1 = self.conv_layer_1(x) #shape of conv_output_1 is 16,300,185\n",
        "        conv_output_2 = self.conv_layer_2(x) #shape of conv_output_2 is 16,300,61\n",
        "        conv_output_3 = self.conv_layer_3(x) #shape of conv_output_3 is 16,300,21\n",
        "        conv_output_linear_1 = self.conv_linear_layer_1(conv_output_1)  #Shape is 16,300,128\n",
        "        conv_output_linear_2 = self.conv_linear_layer_2(conv_output_2)  #Shape is 16,300,128\n",
        "        conv_output_linear_3 = self.conv_linear_layer_3(conv_output_3)  #Shape is 16,300,128\n",
        "        conv_output = torch.cat([conv_output_linear_1,conv_output_linear_2,conv_output_linear_3],\n",
        "                                dim=1)  #Shape is 16,900,128\n",
        "        conv_output = conv_output.permute(0,2,1) #Shape is 16,128,900\n",
        "\n",
        "        linear_output_1 = self.linear_layer_1(conv_output)\n",
        "        linear_output_1 = self.leaky_relu(linear_output_1)\n",
        "        linear_output_1 = self.dropout_layer(linear_output_1)\n",
        "\n",
        "        linear_output_2 = self.linear_layer_2(linear_output_1)\n",
        "        linear_output_2 = self.leaky_relu(linear_output_2)\n",
        "        linear_output_2 = self.dropout_layer(linear_output_2)\n",
        "\n",
        "        linear_output_3 = self.linear_layer_3(linear_output_2)\n",
        "        linear_output_3 = self.leaky_relu(linear_output_3)\n",
        "        linear_output_3 = self.dropout_layer(linear_output_3)\n",
        "\n",
        "        linear_output_4 = self.linear_layer_4(linear_output_3)\n",
        "        linear_output_4 = self.leaky_relu(linear_output_4)\n",
        "        linear_output_4 = self.dropout_layer(linear_output_4)\n",
        "        linear_output_4 = linear_output_4.squeeze(2)\n",
        "\n",
        "        linear_output_5 = self.linear_layer_5(linear_output_4)\n",
        "        linear_output_5 = self.leaky_relu(linear_output_5)\n",
        "        linear_output_5 = self.dropout_layer(linear_output_5)\n",
        "\n",
        "        linear_output_6 = self.linear_layer_6(linear_output_5)\n",
        "        output = linear_output_6\n",
        "        return output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7ypyXg0AmxmC"
      },
      "source": [
        "# Mayank\n",
        "\n",
        "class ReadabilityModel_NeuralNetwork(torch.nn.Module):\n",
        "    def __init__(self, embedding_matrix, hidden_size, num_of_layers):\n",
        "        super().__init__()\n",
        "        self.dropout_probability = 0.2\n",
        "        self.embeddings_layer = torch.nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n",
        "                                                   padding_idx=0)\n",
        "        self.embeddings_layer.weight = torch.nn.Parameter(torch.tensor(embedding_matrix, \n",
        "                                                                       dtype=torch.float32,\n",
        "                                                                       device=device))\n",
        "        self.embeddings_layer.weight.requires_grad = False\n",
        "        self.linear_layer_1 = torch.nn.Linear(300,128)\n",
        "        self.linear_layer_2 = torch.nn.Linear(128,32)\n",
        "        self.linear_layer_3 = torch.nn.Linear(32,8)\n",
        "        self.linear_layer_4 = torch.nn.Linear(8,1)\n",
        "        self.linear_layer_5 = torch.nn.Linear(187,256)\n",
        "        self.linear_layer_6 = torch.nn.Linear(256,512)\n",
        "        self.linear_layer_7 = torch.nn.Linear(512,1024)\n",
        "        self.linear_layer_8 = torch.nn.Linear(1024,256)\n",
        "        self.linear_layer_9 = torch.nn.Linear(256,64)\n",
        "        self.linear_layer_10 = torch.nn.Linear(64,16)\n",
        "        self.linear_layer_11 = torch.nn.Linear(16,5)\n",
        "        self.dropout_layer = torch.nn.Dropout(p=self.dropout_probability)\n",
        "        self.leaky_relu = torch.nn.LeakyReLU()\n",
        "\n",
        "    def forward(self,x):\n",
        "        x = self.embeddings_layer(x) #shape of x is 16,187,300\n",
        "        linear_output_1 = self.linear_layer_1(x)\n",
        "        linear_output_1 = self.leaky_relu(linear_output_1)\n",
        "        linear_output_1 = self.dropout_layer(linear_output_1)\n",
        "\n",
        "        linear_output_2 = self.linear_layer_2(linear_output_1)\n",
        "        linear_output_2 = self.leaky_relu(linear_output_2)\n",
        "        linear_output_2 = self.dropout_layer(linear_output_2)\n",
        "\n",
        "        linear_output_3 = self.linear_layer_3(linear_output_2)\n",
        "        linear_output_3 = self.leaky_relu(linear_output_3)\n",
        "        linear_output_3 = self.dropout_layer(linear_output_3)\n",
        "\n",
        "        linear_output_4 = self.linear_layer_4(linear_output_3)\n",
        "        linear_output_4 = self.leaky_relu(linear_output_4)\n",
        "        linear_output_4 = self.dropout_layer(linear_output_4)\n",
        "        linear_output_4 = linear_output_4.squeeze(2)\n",
        "\n",
        "        linear_output_5 = self.linear_layer_5(linear_output_4)\n",
        "        linear_output_5 = self.leaky_relu(linear_output_5)\n",
        "        linear_output_5 = self.dropout_layer(linear_output_5)\n",
        "\n",
        "        linear_output_6 = self.linear_layer_6(linear_output_5)\n",
        "        linear_output_6 = self.leaky_relu(linear_output_6)\n",
        "        linear_output_6 = self.dropout_layer(linear_output_6)\n",
        "\n",
        "        linear_output_7 = self.linear_layer_7(linear_output_6)\n",
        "        linear_output_7 = self.leaky_relu(linear_output_7)\n",
        "        linear_output_7 = self.dropout_layer(linear_output_7)\n",
        "\n",
        "        linear_output_8 = self.linear_layer_8(linear_output_7)\n",
        "        linear_output_8 = self.leaky_relu(linear_output_8)\n",
        "        linear_output_8 = self.dropout_layer(linear_output_8)\n",
        "\n",
        "        linear_output_9 = self.linear_layer_9(linear_output_8)\n",
        "        linear_output_9 = self.leaky_relu(linear_output_9)\n",
        "        linear_output_9 = self.dropout_layer(linear_output_9)\n",
        "\n",
        "        linear_output_10 = self.linear_layer_10(linear_output_9)\n",
        "        linear_output_10 = self.leaky_relu(linear_output_10)\n",
        "        linear_output_10 = self.dropout_layer(linear_output_10)\n",
        "\n",
        "        linear_output_11 = self.linear_layer_11(linear_output_10)\n",
        "        output = linear_output_11\n",
        "        return output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EpJpUULo8Uzj"
      },
      "source": [
        "## Current"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-17T10:16:38.008865Z",
          "iopub.execute_input": "2021-06-17T10:16:38.009177Z",
          "iopub.status.idle": "2021-06-17T10:16:38.03138Z",
          "shell.execute_reply.started": "2021-06-17T10:16:38.009147Z",
          "shell.execute_reply": "2021-06-17T10:16:38.03056Z"
        },
        "trusted": true,
        "id": "FzjZtmZvC0KU"
      },
      "source": [
        "# mayank\n",
        "\n",
        "class ReadabilityModel_parallel_LSTM_Attention_and_CNN(torch.nn.Module):\n",
        "    def __init__(self, embedding_matrix, hidden_size, num_of_layers):\n",
        "        super().__init__()\n",
        "        self.dropout_prob = 0.4\n",
        "        self.embeddings_layer = torch.nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n",
        "                                                   padding_idx=0)\n",
        "        self.embeddings_layer.weight = torch.nn.Parameter(torch.tensor(embedding_matrix, \n",
        "                                                                       dtype=torch.float32,\n",
        "                                                                       device=device))\n",
        "        self.embeddings_layer.weight.requires_grad = False\n",
        "        \n",
        "        # Block 1 - Lstm layer block\n",
        "        self.lstm_layer_1 = torch.nn.LSTM(embedding_matrix.shape[1], hidden_size,\n",
        "                                        num_layers=num_of_layers,\n",
        "                                        batch_first=True, dropout=self.dropout_prob,\n",
        "                                        bidirectional=True)\n",
        "        self.attention_layer = torch.nn.Linear(hidden_size,hidden_size*2)\n",
        "        self.lstm_linear_layer = torch.nn.Linear(1024,256)\n",
        "        \n",
        "        # Block 2 - Conv layer block\n",
        "        self.conv_layer_1 = torch.nn.Conv1d(in_channels=300,out_channels=300,kernel_size=3,stride=1)\n",
        "        self.conv_layer_2 = torch.nn.Conv1d(in_channels=300,out_channels=300,kernel_size=5,stride=3)\n",
        "        self.conv_layer_3 = torch.nn.Conv1d(in_channels=300,out_channels=300,kernel_size=7,stride=9)\n",
        "        self.conv_linear_layer_1 = torch.nn.Linear(185,128)\n",
        "        self.conv_linear_layer_2 = torch.nn.Linear(61,128)\n",
        "        self.conv_linear_layer_3 = torch.nn.Linear(21,128)\n",
        "        self.conv_output_linear_layer = torch.nn.Linear(900,256)\n",
        "        \n",
        "        # Combining both the blocks \n",
        "        self.linear_layer_1 = torch.nn.Linear(256,64)\n",
        "        self.linear_layer_2 = torch.nn.Linear(64,8)\n",
        "        self.linear_layer_3 = torch.nn.Linear(8,1)\n",
        "        self.linear_layer_4 = torch.nn.Linear(315,128)\n",
        "        self.linear_layer_5 = torch.nn.Linear(128,32)\n",
        "        self.linear_layer_6 = torch.nn.Linear(32,5)\n",
        "        self.tanh = torch.nn.Tanh()\n",
        "        self.relu = torch.nn.ReLU()\n",
        "        self.leaky_relu = torch.nn.LeakyReLU()\n",
        "        self.dropout_layer = torch.nn.Dropout(p=self.dropout_prob)\n",
        "    \n",
        "    def forward(self,x):\n",
        "        x = self.embeddings_layer(x) #shape of x is 16,187,300\n",
        "        \n",
        "        # Block 1 - Lstm layer block\n",
        "        lstm_output_1, (ht1, ct1) = self.lstm_layer_1(x)\n",
        "        #Shape of lstm_output_1 is 16,187,512 \n",
        "        #Shape of ht1 is 6,16,256 \n",
        "        #Shape of ct2 is 6,16,256\n",
        "        #Shape of ht1[-1] is 16,256\n",
        "        attention_output = self.attention_layer(ht1[-1]) #Shape of is 16,512\n",
        "        attention_output = attention_output.unsqueeze(1) #Shape is 16,1,512\n",
        "        attention_context = lstm_output_1 * attention_output #Shape is 16,187,512\n",
        "        softmax_attention = torch.nn.functional.softmax(attention_context,dim=1) #Shape is 16,187,512\n",
        "        word_context = softmax_attention * lstm_output_1  #Shape is 16,187,512\n",
        "        final_context_words = torch.cat([word_context,lstm_output_1],dim=2)  #Shape is 16,187,1024\n",
        "        block_1_output = self.lstm_linear_layer(final_context_words)  #Shape is 16,187,256\n",
        "        block_1_output = self.leaky_relu(block_1_output)\n",
        "        block_1_output = self.dropout_layer(block_1_output)\n",
        "        \n",
        "        # Block 2 - Conv layer block\n",
        "        x_permuted = x.permute(0,2,1)   #Shape is 16,187,200\n",
        "        conv_output_1 = self.conv_layer_1(x_permuted)       #Shape is 16,300,185\n",
        "        conv_output_2 = self.conv_layer_2(x_permuted)       #Shape is 16,300,61\n",
        "        conv_output_3 = self.conv_layer_3(x_permuted)       #Shape is 16,300,21\n",
        "        conv_output_linear_1 = self.conv_linear_layer_1(conv_output_1)  #Shape is 16,300,128\n",
        "        conv_output_linear_2 = self.conv_linear_layer_2(conv_output_2)  #Shape is 16,300,128\n",
        "        conv_output_linear_3 = self.conv_linear_layer_3(conv_output_3)  #Shape is 16,300,128\n",
        "        conv_output = torch.cat([conv_output_linear_1,conv_output_linear_2,conv_output_linear_3],\n",
        "                                dim=1)  #Shape is 16,900,128\n",
        "        conv_output = conv_output.permute(0,2,1) #Shape is 16,128,900\n",
        "        block_2_output = self.conv_output_linear_layer(conv_output)  #Shape is 16,128,256\n",
        "        block_2_output = self.leaky_relu(block_2_output)\n",
        "        block_2_output = self.dropout_layer(block_2_output)\n",
        "\n",
        "        # Combining block 1 & 2\n",
        "        concatenated_output = torch.cat([block_1_output,block_2_output],dim=1)  #Shape is 16,315,256\n",
        "        linear_output_1 = self.linear_layer_1(concatenated_output)  #Shape is 16,315,64\n",
        "        linear_output_1 = self.leaky_relu(linear_output_1)\n",
        "        linear_output_1 = self.dropout_layer(linear_output_1)\n",
        "        \n",
        "        linear_output_2 = self.linear_layer_2(linear_output_1)  #Shape is 16,315,8\n",
        "        linear_output_2 = self.leaky_relu(linear_output_2)\n",
        "        linear_output_2 = self.dropout_layer(linear_output_2)\n",
        "        \n",
        "        linear_output_3 = self.linear_layer_3(linear_output_2)  #Shape is 16,315,1\n",
        "        linear_output_3 = self.dropout_layer(linear_output_3)\n",
        "        linear_output_3 = linear_output_3.squeeze(2)            #Shape is 16,315\n",
        "        \n",
        "        linear_output_4 = self.linear_layer_4(linear_output_3)  #Shape is 16,128\n",
        "        linear_output_4 = self.leaky_relu(linear_output_4)\n",
        "        linear_output_4 = self.dropout_layer(linear_output_4)\n",
        "        \n",
        "        linear_output_5 = self.linear_layer_5(linear_output_4)  #Shape is 16,32\n",
        "        linear_output_5 = self.leaky_relu(linear_output_5)        \n",
        "        linear_output_5 = self.dropout_layer(linear_output_5)\n",
        "        \n",
        "        linear_output_6 = self.linear_layer_6(linear_output_5)  #Shape is 16,5\n",
        "        output = linear_output_6\n",
        "        return output\n",
        "\n",
        "# Average Validation Loss:\n",
        "# Lowest validation loss is  at fold 3"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0fxt1r0eIKHB"
      },
      "source": [
        "## To be done"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-06T13:45:54.561125Z",
          "iopub.execute_input": "2021-06-06T13:45:54.561523Z",
          "iopub.status.idle": "2021-06-06T13:45:54.576383Z",
          "shell.execute_reply.started": "2021-06-06T13:45:54.561489Z",
          "shell.execute_reply": "2021-06-06T13:45:54.574846Z"
        },
        "trusted": true,
        "id": "f2l13b1hC0KN"
      },
      "source": [
        "# Mayank\n",
        "\n",
        "class ReadabilityModel_CNN_LSTM_Attention(torch.nn.Module):\n",
        "    def __init__(self, embedding_matrix, hidden_size, num_of_layers):\n",
        "        super().__init__()\n",
        "        self.dropout_probability = 0.2\n",
        "        self.embeddings_layer = torch.nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n",
        "                                                   padding_idx=0)\n",
        "        self.embeddings_layer.weight = torch.nn.Parameter(torch.tensor(embedding_matrix, \n",
        "                                                                       dtype=torch.float32,\n",
        "                                                                       device=device))\n",
        "        self.embeddings_layer.weight.requires_grad = False\n",
        "        self.conv_layer_1 = torch.nn.Conv1d(in_channels=300,out_channels=300,kernel_size=3,stride=1)\n",
        "        self.conv_layer_2 = torch.nn.Conv1d(in_channels=300,out_channels=300,kernel_size=5,stride=3)\n",
        "        self.lstm_layer_1 = torch.nn.LSTM(300, hidden_size,\n",
        "                                        num_layers=num_of_layers, dropout=self.dropout_probability,\n",
        "                                        batch_first=True, bidirectional=True)\n",
        "        self.attention_layer = torch.nn.Linear(hidden_size, hidden_size*2)\n",
        "        self.linear_layer_1 = torch.nn.Linear(512,256)\n",
        "        self.linear_layer_2 = torch.nn.Linear(256,64)\n",
        "        self.linear_layer_3 = torch.nn.Linear(64,16)\n",
        "        self.linear_layer_4 = torch.nn.Linear(16,1)\n",
        "        self.dropout_layer = torch.nn.Dropout(p=self.dropout_probability)\n",
        "        self.leaky_relu = torch.nn.LeakyReLU()       \n",
        "        self.output_layer = torch.nn.Linear(50,5)\n",
        "    \n",
        "    def forward(self,x):\n",
        "        x = self.embeddings_layer(x) #shape of x is 16,187,300\n",
        "        conv_output_1 = self.conv_layer_1(x) #shape of conv_output_1 is 16,100,300\n",
        "        conv_output_2 = self.conv_layer_2(conv_output_1) #shape of conv_output_2 is 16,50,300\n",
        "        lstm_output_1, (ht1, ct1) = self.lstm_layer_1(conv_output_2)\n",
        "        #Shape of lstm_output_1 is 16,50,512 \n",
        "        #Shape of ht1 is 6,16,256 \n",
        "        #Shape of ct2 is 6,16,256\n",
        "        attention_output = self.attention_layer(ht1[-1])\n",
        "        attention_output = attention_output.unsqueeze(1) #Shape is 16,1,512\n",
        "        attention_context = lstm_output_1 * attention_output #Shape is 16,50,512\n",
        "        softmax_attention = torch.nn.functional.softmax(attention_context,dim=1) #Shape is 16,50,512\n",
        "        word_context = softmax_attention * lstm_output_1  #Shape is 16,50,512\n",
        "#         final_context_words = torch.cat([word_context,lstm_output_1],dim=2)  #Shape is 16,50,1024\n",
        "        linear_output_1 = self.linear_layer_1(word_context)\n",
        "        linear_output_1 = self.leaky_relu(linear_output_1)\n",
        "        linear_output_1 = self.dropout_layer(linear_output_1)\n",
        "\n",
        "        linear_output_2 = self.linear_layer_2(linear_output_1)\n",
        "        linear_output_2 = self.leaky_relu(linear_output_2)\n",
        "        linear_output_2 = self.dropout_layer(linear_output_2)\n",
        "\n",
        "        linear_output_3 = self.linear_layer_3(linear_output_2)\n",
        "        linear_output_3 = self.leaky_relu(linear_output_3)\n",
        "        linear_output_3 = self.dropout_layer(linear_output_3)\n",
        "\n",
        "        linear_output_4 = self.linear_layer_4(linear_output_3)\n",
        "        linear_output_4 = linear_output_4.squeeze(2)\n",
        "        output = self.output_layer(linear_output_4)\n",
        "        return output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ETUEILot8TH1"
      },
      "source": [
        "# talha\n",
        "\n",
        "class ReadabilityModel_LSTM_Attention(torch.nn.Module):\n",
        "    def __init__(self, embedding_matrix, hidden_size, num_of_layers):\n",
        "        super().__init__()\n",
        "        self.dropout_probability = 0.2\n",
        "        self.embeddings_layer = torch.nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n",
        "                                                   padding_idx=0)\n",
        "        self.embeddings_layer.weight = torch.nn.Parameter(torch.tensor(embedding_matrix, \n",
        "                                                                       dtype=torch.float32,\n",
        "                                                                       device=device))\n",
        "        self.embeddings_layer.weight.requires_grad = False\n",
        "        self.lstm_layer_1 = torch.nn.LSTM(300, hidden_size,\n",
        "                                        num_layers=num_of_layers, dropout=self.dropout_probability,\n",
        "                                        batch_first=True, bidirectional=True)\n",
        "        self.attention_layer = torch.nn.Linear(hidden_size, hidden_size*2)\n",
        "        self.linear_layer_1 = torch.nn.Linear(512,256)\n",
        "        self.linear_layer_2 = torch.nn.Linear(256,64)\n",
        "        self.linear_layer_3 = torch.nn.Linear(64,16)\n",
        "        self.linear_layer_4 = torch.nn.Linear(16,1)\n",
        "        self.dropout_layer = torch.nn.Dropout(p=self.dropout_probability)\n",
        "        self.leaky_relu = torch.nn.LeakyReLU()\n",
        "        self.output_layer = torch.nn.Linear(187,5)\n",
        "    \n",
        "    def forward(self,x):\n",
        "        x = self.embeddings_layer(x) #shape of x is 16,200,300\n",
        "        lstm_output_1, (ht1, ct1) = self.lstm_layer_1(x)\n",
        "        #Shape of lstm_output_1 is 16,200,512 \n",
        "        #Shape of ht1 is 6,16,256 \n",
        "        #Shape of ct2 is 6,16,256\n",
        "        attention_output = self.attention_layer(ht1[-1])\n",
        "        attention_output = attention_output.unsqueeze(1) #Shape is 16,1,512\n",
        "        attention_context = lstm_output_1 * attention_output #Shape is 16,187,512\n",
        "        softmax_attention = torch.nn.functional.softmax(attention_context,dim=1) #Shape is 16,50,512\n",
        "        word_context = softmax_attention * lstm_output_1  #Shape is 16,50,512\n",
        "#         final_context_words = torch.cat([word_context,lstm_output_1],dim=2)  #Shape is 16,50,1024\n",
        "        linear_output_1 = self.linear_layer_1(word_context)\n",
        "        linear_output_1 = self.leaky_relu(linear_output_1)\n",
        "        linear_output_1 = self.dropout_layer(linear_output_1)\n",
        "        linear_output_2 = self.linear_layer_2(linear_output_1)\n",
        "        linear_output_2 = self.leaky_relu(linear_output_2)\n",
        "        linear_output_2 = self.dropout_layer(linear_output_2)\n",
        "        linear_output_3 = self.linear_layer_3(linear_output_2)\n",
        "        linear_output_3 = self.leaky_relu(linear_output_3)\n",
        "        linear_output_3 = self.dropout_layer(linear_output_3)\n",
        "        linear_output_4 = self.linear_layer_4(linear_output_3)\n",
        "        linear_output_4 = self.leaky_relu(linear_output_4)\n",
        "        linear_output_4 = self.dropout_layer(linear_output_4)\n",
        "        linear_output_4 = linear_output_4.squeeze(2)\n",
        "        output = self.output_layer(linear_output_4)\n",
        "        return output\n",
        "\n",
        "# Average Validation Loss:0.7848541466199805\n",
        "# Lowest Validation loss is 0.7264222720253899 at fold 3"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l_M-nxJEC0KO"
      },
      "source": [
        "# Talha\n",
        "\n",
        "class ReadabilityModel_CNN_LSTM_Attention_CNN(torch.nn.Module):\n",
        "    def __init__(self, embedding_matrix, hidden_size, num_of_layers, batch_size):\n",
        "        super().__init__()\n",
        "        self.embeddings_layer = torch.nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n",
        "                                                   padding_idx=0)\n",
        "        self.embeddings_layer.weight = torch.nn.Parameter(torch.tensor(embedding_matrix, \n",
        "                                                                       dtype=torch.float32,\n",
        "                                                                       device=device))\n",
        "        self.embeddings_layer.weight.requires_grad = False\n",
        "        self.conv_layer_1 = torch.nn.Conv1d(in_channels=200,out_channels=100,kernel_size=1,stride=1)\n",
        "        self.conv_layer_2 = torch.nn.Conv1d(in_channels=100,out_channels=50,kernel_size=1,stride=1)\n",
        "        self.lstm_layer_1 = torch.nn.LSTM(300, hidden_size,\n",
        "                                        num_layers=num_of_layers,\n",
        "                                        batch_first=True, bidirectional=True)\n",
        "        self.attention_layer = torch.nn.Linear(hidden_size, hidden_size*2)\n",
        "        self.linear_layer_1 = torch.nn.Linear(512,256)\n",
        "        self.linear_layer_2 = torch.nn.Linear(256,64)\n",
        "        self.linear_layer_3 = torch.nn.Linear(64,16)\n",
        "        self.linear_layer_4 = torch.nn.Linear(16,1)       \n",
        "        self.output_layer = torch.nn.Linear(50,1)\n",
        "    \n",
        "    def forward(self,x):\n",
        "        x = self.embeddings_layer(x) #shape of x is 16,200,300\n",
        "        conv_output_1 = self.conv_layer_1(x) #shape of conv_output_1 is 16,100,300\n",
        "        conv_output_2 = self.conv_layer_2(conv_output_1) #shape of conv_output_2 is 16,50,300\n",
        "        lstm_output_1, (ht1, ct1) = self.lstm_layer_1(conv_output_2)\n",
        "        #Shape of lstm_output_1 is 16,50,512 \n",
        "        #Shape of ht1 is 6,16,256 \n",
        "        #Shape of ct2 is 6,16,256\n",
        "        attention_output = self.attention_layer(ht1[-1])\n",
        "        attention_output = attention_output.unsqueeze(1) #Shape is 16,1,512\n",
        "        attention_context = lstm_output_1 * attention_output #Shape is 16,50,512\n",
        "        softmax_attention = torch.nn.functional.softmax(attention_context,dim=1) #Shape is 16,50,512\n",
        "        word_context = softmax_attention * lstm_output_1  #Shape is 16,50,512\n",
        "#         final_context_words = torch.cat([word_context,lstm_output_1],dim=2)  #Shape is 16,50,1024\n",
        "        linear_output_1 = self.linear_layer_1(word_context)\n",
        "        linear_output_2 = self.linear_layer_2(linear_output_1)\n",
        "        linear_output_3 = self.linear_layer_3(linear_output_2)\n",
        "        linear_output_4 = self.linear_layer_4(linear_output_3)\n",
        "        linear_output_4 = linear_output_4.squeeze(2)\n",
        "        output = self.output_layer(linear_output_4)\n",
        "        return output\n",
        "\n",
        "# Average Validation Loss:0.7848541466199805\n",
        "# Lowest Validation loss is 0.7264222720253899 at fold 3"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-06T11:18:35.92226Z",
          "iopub.execute_input": "2021-06-06T11:18:35.925335Z",
          "iopub.status.idle": "2021-06-06T11:18:35.964619Z",
          "shell.execute_reply.started": "2021-06-06T11:18:35.925277Z",
          "shell.execute_reply": "2021-06-06T11:18:35.96208Z"
        },
        "trusted": true,
        "id": "E4_gghJNC0KP"
      },
      "source": [
        "# Mayank\n",
        "\n",
        "class ReadabilityModel_CNN_LSTM_Attention_LSTM(torch.nn.Module):\n",
        "    def __init__(self, embedding_matrix, hidden_size, num_of_layers, batch_size):\n",
        "        super().__init__()\n",
        "        self.embeddings_layer = torch.nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n",
        "                                                   padding_idx=0)\n",
        "        self.embeddings_layer.weight = torch.nn.Parameter(torch.tensor(embedding_matrix, \n",
        "                                                                       dtype=torch.float32,\n",
        "                                                                       device=device))\n",
        "        self.embeddings_layer.weight.requires_grad = False\n",
        "        self.conv_layer_1 = torch.nn.Conv1d(in_channels=200,out_channels=100,kernel_size=1,stride=1)\n",
        "        self.conv_layer_2 = torch.nn.Conv1d(in_channels=100,out_channels=50,kernel_size=1,stride=1)\n",
        "        self.lstm_layer_1 = torch.nn.LSTM(300, hidden_size,\n",
        "                                        num_layers=num_of_layers,\n",
        "                                        batch_first=True, bidirectional=True)\n",
        "        self.attention_layer = torch.nn.Linear(hidden_size, hidden_size*2)\n",
        "        self.linear_layer_embedding_context = torch.nn.Linear(1024,512)\n",
        "        self.lstm_layer_2 = torch.nn.LSTM(512, hidden_size,\n",
        "                                        num_layers=num_of_layers,\n",
        "                                        batch_first=True, bidirectional=True)\n",
        "        self.linear_layer_1 = torch.nn.Linear(256,128)\n",
        "        self.linear_layer_2 = torch.nn.Linear(128,32)\n",
        "        self.linear_layer_3 = torch.nn.Linear(32,8)\n",
        "        self.linear_layer_4 = torch.nn.Linear(8,1)\n",
        "        self.dropout_layer = torch.nn.Dropout(p=0.1)\n",
        "    \n",
        "    def forward(self,x):\n",
        "        x = self.embeddings_layer(x) #shape of x is 16,200,300\n",
        "        conv_output_1 = self.conv_layer_1(x) #shape of conv_output_1 is 16,100,300\n",
        "        conv_output_1 = self.dropout_layer(conv_output_1)\n",
        "        conv_output_2 = self.conv_layer_2(conv_output_1) #shape of conv_output_2 is 16,50,300\n",
        "        lstm_output_1, (ht1, ct1) = self.lstm_layer_1(conv_output_2)\n",
        "        #Shape of lstm_output_1 is 16,50,512 \n",
        "        #Shape of ht1 is 6,16,256 \n",
        "        #Shape of ct2 is 6,16,256\n",
        "        attention_output = self.attention_layer(ht1[-1])\n",
        "        attention_output = attention_output.unsqueeze(1) #Shape is 16,1,512\n",
        "        attention_context = lstm_output_1 * attention_output #Shape is 16,50,512\n",
        "        attention_context = self.dropout_layer(attention_context)\n",
        "        softmax_attention = torch.nn.functional.softmax(attention_context,dim=1) #Shape is 16,50,512\n",
        "        word_context = softmax_attention * lstm_output_1  #Shape is 16,50,512\n",
        "        final_context_words = word_context*lstm_output_1  #Shape is 16,50,512\n",
        "        lstm_output_2, (ht2, ct2) = self.lstm_layer_2(final_context_words)\n",
        "        #Shape of lstm_output_2 is 16,50,1024 \n",
        "        #Shape of ht2 is 6,16,256 \n",
        "        #Shape of ct2 is 6,16,256\n",
        "        linear_output_1 = self.linear_layer_1(ht2[-1])     #Shape is 16,128\n",
        "        linear_output_2 = self.linear_layer_2(linear_output_1)   #Shape is 16,32\n",
        "        linear_output_2 = self.dropout_layer(linear_output_2)\n",
        "        linear_output_3 = self.linear_layer_3(linear_output_2)   #Shape is 16,8\n",
        "        linear_output_4 = self.linear_layer_4(linear_output_3)   #Shape is 16,1\n",
        "        output = linear_output_4   #Shape of output is 16,1\n",
        "        return output\n",
        "    \n",
        "# Average Validation Loss:0.9133829620398682 \n",
        "# Lowest Validation Loss: 0.8187914316807785 at fold 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-12T15:39:47.24523Z",
          "iopub.execute_input": "2021-06-12T15:39:47.245841Z",
          "iopub.status.idle": "2021-06-12T15:39:47.264813Z",
          "shell.execute_reply.started": "2021-06-12T15:39:47.245786Z",
          "shell.execute_reply": "2021-06-12T15:39:47.264052Z"
        },
        "trusted": true,
        "id": "dfpZBqlSC0KQ"
      },
      "source": [
        "# talha\n",
        "\n",
        "class ReadabilityModel_LSTM_Attention_CNN(torch.nn.Module):\n",
        "    def __init__(self, embedding_matrix, hidden_size, num_of_layers, batch_size, training):\n",
        "        super().__init__()\n",
        "        if training:\n",
        "            self.dropout_prob = 0.2\n",
        "        else:\n",
        "            self.dropout_prop = 0.0\n",
        "        self.embeddings_layer = torch.nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n",
        "                                                   padding_idx=0)\n",
        "        self.embeddings_layer.weight = torch.nn.Parameter(torch.tensor(embedding_matrix, \n",
        "                                                                       dtype=torch.float32,\n",
        "                                                                       device=device))\n",
        "        self.embeddings_layer.weight.requires_grad = False\n",
        "        self.lstm_layer_1 = torch.nn.LSTM(embedding_matrix.shape[1], hidden_size,\n",
        "                                        num_layers=num_of_layers,\n",
        "                                        batch_first=True, dropout=self.dropout_prob,\n",
        "                                        bidirectional=True)\n",
        "        self.attention_layer = torch.nn.Linear(hidden_size,hidden_size*2)\n",
        "        self.conv_layer_1 = torch.nn.Conv1d(in_channels=1024,out_channels=1024,kernel_size=3,stride=1)\n",
        "        self.conv_layer_2 = torch.nn.Conv1d(in_channels=1024,out_channels=1024,kernel_size=5,stride=3)\n",
        "        self.conv_layer_3 = torch.nn.Conv1d(in_channels=1024,out_channels=1024,kernel_size=11,stride=9)\n",
        "        self.conv_linear_layer_1 = torch.nn.Linear(198,128)\n",
        "        self.conv_linear_layer_2 = torch.nn.Linear(66,128)\n",
        "        self.conv_linear_layer_3 = torch.nn.Linear(22,128)\n",
        "        self.conv_output_linear_layer_1 = torch.nn.Linear(3072,1024)\n",
        "        self.conv_output_linear_layer_2 = torch.nn.Linear(1024,256)\n",
        "        self.conv_output_linear_layer_3 = torch.nn.Linear(256,32)\n",
        "        self.conv_output_linear_layer_4 = torch.nn.Linear(32,8)\n",
        "        self.tanh = torch.nn.Tanh()\n",
        "        self.conv_output_linear_layer_5 = torch.nn.Linear(8,1)\n",
        "        self.linear_layer_1 = torch.nn.Linear(128,64)\n",
        "        self.linear_layer_2 = torch.nn.Linear(64,1)\n",
        "    \n",
        "    def forward(self,x):\n",
        "        x = self.embeddings_layer(x) #shape of x is 16,200,300\n",
        "        lstm_output_1, (ht1, ct1) = self.lstm_layer_1(x)\n",
        "        #Shape of lstm_output_1 is 16,200,512 \n",
        "        #Shape of ht1 is 6,16,256 \n",
        "        #Shape of ct2 is 6,16,256\n",
        "        #Shape of ht1[-1] is 16,256\n",
        "        attention_output = self.attention_layer(ht1[-1]) #Shape of is 16,512\n",
        "        attention_output = attention_output.unsqueeze(1) #Shape is 16,1,512\n",
        "        attention_context = lstm_output_1 * attention_output #Shape is 16,200,512\n",
        "        softmax_attention = torch.nn.functional.softmax(attention_context,dim=1) #Shape is 16,200,512\n",
        "        word_context = softmax_attention * lstm_output_1  #Shape is 16,200,512\n",
        "        final_context_words = torch.cat([word_context,lstm_output_1],dim=2)  #Shape is 16,200,1024\n",
        "        final_context_words = final_context_words.permute(0,2,1)\n",
        "        conv_output_1 = self.conv_layer_1(final_context_words)       #Shape is 16,1024,198\n",
        "        conv_output_2 = self.conv_layer_2(final_context_words)       #Shape is 16,1024,66\n",
        "        conv_output_3 = self.conv_layer_3(final_context_words)       #Shape is 16,1024,22\n",
        "        conv_output_linear_1 = self.conv_linear_layer_1(conv_output_1)  #Shape is 16,1024,128\n",
        "        conv_output_linear_2 = self.conv_linear_layer_2(conv_output_2)  #Shape is 16,1024,128\n",
        "        conv_output_linear_3 = self.conv_linear_layer_3(conv_output_3)  #Shape is 16,1024,128\n",
        "        conv_output = torch.cat([conv_output_linear_1,conv_output_linear_2,conv_output_linear_3],\n",
        "                                dim=1)  #Shape is 16,3072,128\n",
        "        conv_output = conv_output.permute(0,2,1) #Shape is 16,128,3072\n",
        "        conv_linear_output_1 = self.conv_output_linear_layer_1(conv_output)  #Shape is 16,128,1024\n",
        "        conv_linear_output_2 = self.conv_output_linear_layer_2(conv_linear_output_1)#Shape is 16,128,256\n",
        "        conv_linear_output_2 = self.tanh(conv_linear_output_2)\n",
        "        conv_linear_output_3 = self.conv_output_linear_layer_3(conv_linear_output_2)\n",
        "        conv_linear_output_4 = self.conv_output_linear_layer_4(conv_linear_output_3)\n",
        "        conv_linear_output_4 = self.tanh(conv_linear_output_4)\n",
        "        conv_linear_output_5 = self.conv_output_linear_layer_5(conv_linear_output_4)\n",
        "        conv_linear_output_5 = conv_linear_output_5.squeeze(2)\n",
        "        linear_output_1 = self.linear_layer_1(conv_linear_output_5)  #Shape is 16,64\n",
        "        linear_output_1 = self.tanh(linear_output_1)\n",
        "        linear_output_2 = self.linear_layer_2(linear_output_1)  #Shape is 16,1\n",
        "        output = linear_output_2   #Shape of output is 16,1\n",
        "        return output\n",
        "\n",
        "# Average Validation Loss:0.6796757201937529\n",
        "# Lowest validation loss is 0.6500918945654444 at fold 3"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-09T03:23:10.046616Z",
          "iopub.execute_input": "2021-06-09T03:23:10.047071Z",
          "iopub.status.idle": "2021-06-09T03:23:10.081609Z",
          "shell.execute_reply.started": "2021-06-09T03:23:10.047033Z",
          "shell.execute_reply": "2021-06-09T03:23:10.080605Z"
        },
        "trusted": true,
        "id": "FhFL925rC0KS"
      },
      "source": [
        "# talha\n",
        "\n",
        "class ReadabilityModel_LSTM_Attention_CNN_GRU(torch.nn.Module):\n",
        "    def __init__(self, embedding_matrix, hidden_size, num_of_layers, batch_size, training):\n",
        "        super().__init__()\n",
        "        if training:\n",
        "            self.dropout_prob = 0.2\n",
        "        else:\n",
        "            self.dropout_prop = 0.0\n",
        "        self.embeddings_layer = torch.nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n",
        "                                                   padding_idx=0)\n",
        "        self.embeddings_layer.weight = torch.nn.Parameter(torch.tensor(embedding_matrix, \n",
        "                                                                       dtype=torch.float32,\n",
        "                                                                       device=device))\n",
        "        self.embeddings_layer.weight.requires_grad = False\n",
        "        self.lstm_layer_1 = torch.nn.LSTM(embedding_matrix.shape[1], hidden_size,\n",
        "                                        num_layers=num_of_layers,\n",
        "                                        batch_first=True, dropout=self.dropout_prob,\n",
        "                                        bidirectional=True)\n",
        "        self.attention_layer = torch.nn.Linear(hidden_size,hidden_size*2)\n",
        "        self.conv_layer_1 = torch.nn.Conv1d(in_channels=1024,out_channels=1024,kernel_size=3,stride=1)\n",
        "        self.conv_layer_2 = torch.nn.Conv1d(in_channels=1024,out_channels=1024,kernel_size=5,stride=3)\n",
        "        self.conv_layer_3 = torch.nn.Conv1d(in_channels=1024,out_channels=1024,kernel_size=11,stride=9)\n",
        "        self.conv_linear_layer_1 = torch.nn.Linear(198,128)\n",
        "        self.conv_linear_layer_2 = torch.nn.Linear(66,128)\n",
        "        self.conv_linear_layer_3 = torch.nn.Linear(22,128)\n",
        "        self.conv_output_linear_layer_1 = torch.nn.Linear(3072,1024)\n",
        "        self.conv_output_linear_layer_2 = torch.nn.Linear(1024,256)\n",
        "        self.gru_layer = torch.nn.GRU(256,hidden_size,num_layers=num_of_layers,batch_first=True,\n",
        "                                     dropout=self.dropout_prob)\n",
        "        self.linear_layer_1 = torch.nn.Linear(256,64)\n",
        "        self.linear_layer_2 = torch.nn.Linear(64,8)\n",
        "        self.linear_layer_3 = torch.nn.Linear(8,1)\n",
        "    \n",
        "    def forward(self,x):\n",
        "        x = self.embeddings_layer(x) #shape of x is 16,200,300\n",
        "        lstm_output_1, (ht1, ct1) = self.lstm_layer_1(x)\n",
        "        #Shape of lstm_output_1 is 16,200,512 \n",
        "        #Shape of ht1 is 6,16,256 \n",
        "        #Shape of ct2 is 6,16,256\n",
        "        #Shape of ht1[-1] is 16,256\n",
        "        attention_output = self.attention_layer(ht1[-1]) #Shape of is 16,512\n",
        "        attention_output = attention_output.unsqueeze(1) #Shape is 16,1,512\n",
        "        attention_context = lstm_output_1 * attention_output #Shape is 16,200,512\n",
        "        softmax_attention = torch.nn.functional.softmax(attention_context,dim=1) #Shape is 16,200,512\n",
        "        word_context = softmax_attention * lstm_output_1  #Shape is 16,200,512\n",
        "        final_context_words = torch.cat([word_context,lstm_output_1],dim=2)  #Shape is 16,200,1024\n",
        "        final_context_words = final_context_words.permute(0,2,1)\n",
        "        conv_output_1 = self.conv_layer_1(final_context_words)       #Shape is 16,1024,198\n",
        "        conv_output_2 = self.conv_layer_2(final_context_words)       #Shape is 16,1024,66\n",
        "        conv_output_3 = self.conv_layer_3(final_context_words)       #Shape is 16,1024,22\n",
        "        conv_output_linear_1 = self.conv_linear_layer_1(conv_output_1)  #Shape is 16,1024,128\n",
        "        conv_output_linear_2 = self.conv_linear_layer_2(conv_output_2)  #Shape is 16,1024,128\n",
        "        conv_output_linear_3 = self.conv_linear_layer_3(conv_output_3)  #Shape is 16,1024,128\n",
        "        conv_output = torch.cat([conv_output_linear_1,conv_output_linear_2,conv_output_linear_3],\n",
        "                                dim=1)  #Shape is 16,3072,128\n",
        "        conv_output = conv_output.permute(0,2,1) #Shape is 16,128,3072\n",
        "        conv_linear_output_1 = self.conv_output_linear_layer_1(conv_output)  #Shape is 16,128,1024\n",
        "        conv_linear_output_2 = self.conv_output_linear_layer_2(conv_linear_output_1)#Shape is 16,128,256\n",
        "        gru_output , ht = self.gru_layer(conv_linear_output_2)\n",
        "        #Shape of ht is 16,256\n",
        "        linear_output_1 = self.linear_layer_1(ht[-1])  #Shape is 16,64\n",
        "        linear_output_2 = self.linear_layer_2(linear_output_1)  #Shape is 16,8\n",
        "        linear_output_3 = self.linear_layer_3(linear_output_2)  #Shape is 16,1\n",
        "        output = linear_output_3   #Shape of output is 16,1\n",
        "        return output\n",
        "\n",
        "# Average Validation Loss:1.0334918799897828\n",
        "# Lowest avalidation loss is 1.013418031971577 at fold 3"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-07T14:56:01.375534Z",
          "iopub.execute_input": "2021-06-07T14:56:01.376153Z",
          "iopub.status.idle": "2021-06-07T14:56:01.401621Z",
          "shell.execute_reply.started": "2021-06-07T14:56:01.376099Z",
          "shell.execute_reply": "2021-06-07T14:56:01.400629Z"
        },
        "trusted": true,
        "id": "AroKhpBlC0KT"
      },
      "source": [
        "# mayank\n",
        "\n",
        "class CommonLitCNNLSTMAttention_EnsembleModel(torch.nn.Module):\n",
        "    def __init__(self, embedding_matrix, hidden_size, num_layers, batch_size):\n",
        "        super().__init__()\n",
        "        self.batch_size = batch_size\n",
        "        self.hidden_size = hidden_size\n",
        "        self.num_layers = num_layers\n",
        "                \n",
        "        self.embedding_layer = torch.nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n",
        "                                                   padding_idx=0)\n",
        "        self.embedding_layer.weight = torch.nn.Parameter(torch.tensor(embedding_matrix,\n",
        "                                                                      dtype=torch.float32,\n",
        "                                                                      device=device))\n",
        "        self.embedding_layer.weight.requires_grad = False\n",
        "        \n",
        "        # Block 1\n",
        "        self.lstm_layer_1 = torch.nn.LSTM(embedding_matrix.shape[1],hidden_size,\n",
        "                                          batch_first = True,num_layers = self.num_layers,\n",
        "                                          bidirectional=True)        \n",
        "        self.attention_linear_layer = torch.nn.Linear(hidden_size,2*hidden_size)\n",
        "        self.lstm_layer_2 = torch.nn.LSTM(4*hidden_size,hidden_size,\n",
        "                                          batch_first=True,num_layers = self.num_layers,\n",
        "                                          bidirectional=True)\n",
        "        \n",
        "        # Block 2\n",
        "        self.lstm_layer_3 = torch.nn.LSTM(embedding_matrix.shape[1],hidden_size,\n",
        "                                    batch_first = True,num_layers = self.num_layers,\n",
        "                                    bidirectional=True)        \n",
        "        self.conv1 = torch.nn.Conv1d(in_channels = 2*hidden_size, out_channels=hidden_size,\n",
        "                               kernel_size=3,stride=1)\n",
        "        self.conv2 = torch.nn.Conv1d(in_channels = 2*hidden_size, out_channels=hidden_size,\n",
        "                               kernel_size=5,stride=1)\n",
        "        self.conv3 = torch.nn.Conv1d(in_channels = 2*hidden_size, out_channels=hidden_size,\n",
        "                               kernel_size=7,stride=1)\n",
        "        self.low_lstm = torch.nn.LSTM(hidden_size,hidden_size,\n",
        "                                batch_first = True,num_layers = self.num_layers,\n",
        "                                bidirectional=True)\n",
        "        self.med_lstm = torch.nn.LSTM(hidden_size,hidden_size,\n",
        "                                batch_first = True,num_layers = self.num_layers,\n",
        "                                bidirectional=True)\n",
        "        self.high_lstm = torch.nn.LSTM(hidden_size,hidden_size,\n",
        "                                 batch_first = True,num_layers = self.num_layers,\n",
        "                                 bidirectional=True)\n",
        "        self.lstm_features_concat_layer = torch.nn.Linear(3*hidden_size,hidden_size)\n",
        "\n",
        "        #Combining Block 1 and 2\n",
        "        self.output_linear_1 = torch.nn.Linear(2*hidden_size,hidden_size)\n",
        "        self.output_linear_2 = torch.nn.Linear(hidden_size,hidden_size // 2)\n",
        "        self.output_linear_3 = torch.nn.Linear(hidden_size// 2,1)\n",
        "    \n",
        "    \n",
        "    def forward(self,input_text):\n",
        "        self.embeddings = self.embedding_layer(input_text.long().to(device))\n",
        "        self.embeddings = self.embeddings.permute(0, 2, 1)   # convert to [batch, channels, time]\n",
        "        self.embeddings = torch.nn.functional.dropout2d(self.embeddings, 0.2, training=self.training)\n",
        "        self.embeddings = self.embeddings.permute(0, 2, 1)\n",
        "        \n",
        "        #Block 1\n",
        "        lstm_output_1,(hidden_state_1,cell_state) = self.lstm_layer_1(self.embeddings)\n",
        "        final_state_1 = hidden_state_1[-1,:,:]\n",
        "        attention_linear_output = self.attention_linear_layer(final_state_1)\n",
        "        attention_linear_output = attention_linear_output.unsqueeze(1)\n",
        "        attention_multiplied_context = lstm_output_1 * attention_linear_output\n",
        "        softmax_attention = torch.nn.functional.softmax(attention_multiplied_context,dim=1)\n",
        "        global_context = softmax_attention * lstm_output_1\n",
        "        final_context_words = torch.cat([global_context,lstm_output_1],dim=2) # 64,seq_len,4*hidden_size\n",
        "        lstm_output_2, (hidden_state_2,cell_state_2) = self.lstm_layer_2(final_context_words)\n",
        "        final_state_2 = hidden_state_2[-1,:,:]\n",
        "\n",
        "        #Block 2\n",
        "        lstm_output_3,(hidden_state_3,cell_state_3) = self.lstm_layer_3(self.embeddings)\n",
        "        lstm_output_3 = lstm_output_3.permute(0,2,1)\n",
        "        \n",
        "        conv_1_output = self.conv1(lstm_output_3)\n",
        "        conv_1_output = conv_1_output.permute(0,2,1) \n",
        "\n",
        "        conv_2_output = self.conv2(lstm_output_3)\n",
        "        conv_2_output = conv_2_output.permute(0,2,1)\n",
        "        \n",
        "        conv_3_output = self.conv3(lstm_output_3)\n",
        "        conv_3_output = conv_3_output.permute(0,2,1)\n",
        "\n",
        "        low_lstm_output,(hidden_state_low,cell_state_low) = self.low_lstm(conv_1_output)\n",
        "        med_lstm_output,(hidden_state_med,cell_state_med) = self.med_lstm(conv_2_output)\n",
        "        high_lstm_output,(hidden_state_high,cell_state_high) = self.high_lstm(conv_3_output)\n",
        "        concat_features = torch.cat([hidden_state_low[-1,:,:],hidden_state_med[-1,:,:],hidden_state_high[-1,:,:]],dim=1)\n",
        "        lstm_linear_concat_output = self.lstm_features_concat_layer(concat_features)\n",
        "\n",
        "        #Combining BLock 1 and 2 \n",
        "        short_long_context_features = torch.cat([final_state_2,lstm_linear_concat_output],dim=1)\n",
        "        linear_output_1 = self.output_linear_1(short_long_context_features)\n",
        "        linear_output_2 = self.output_linear_2(linear_output_1)\n",
        "        linear_output_3 = self.output_linear_3(linear_output_2)\n",
        "\n",
        "        return linear_output_3"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-17T13:07:03.522849Z",
          "iopub.execute_input": "2021-06-17T13:07:03.523228Z",
          "iopub.status.idle": "2021-06-17T13:07:03.546108Z",
          "shell.execute_reply.started": "2021-06-17T13:07:03.523197Z",
          "shell.execute_reply": "2021-06-17T13:07:03.544794Z"
        },
        "trusted": true,
        "id": "PMCdF2WAC0KU",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 231
        },
        "outputId": "7d9f3c34-275c-45a5-fa80-19749118e382"
      },
      "source": [
        "# talha\n",
        "\n",
        "class ReadabilityModel_parallel_LSTM_Attention_and_LSTM_CNN(torch.nn.Module):\n",
        "    def __init__(self, embedding_matrix, hidden_size, num_of_layers, batch_size, training):\n",
        "        super().__init__()\n",
        "        if training:\n",
        "            self.dropout_prob = 0.2\n",
        "        else:\n",
        "            self.dropout_prop = 0.0\n",
        "        self.embeddings_layer = torch.nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n",
        "                                                   padding_idx=0)\n",
        "        self.embeddings_layer.weight = torch.nn.Parameter(torch.tensor(embedding_matrix, \n",
        "                                                                       dtype=torch.float32,\n",
        "                                                                       device=device))\n",
        "        self.embeddings_layer.weight.requires_grad = False\n",
        "        \n",
        "        # Block 1 - Lstm layer block\n",
        "        self.lstm_layer_1 = torch.nn.LSTM(embedding_matrix.shape[1], hidden_size,\n",
        "                                        num_layers=num_of_layers,\n",
        "                                        batch_first=True, dropout=self.dropout_prob,\n",
        "                                        bidirectional=True)\n",
        "        self.attention_layer = torch.nn.Linear(hidden_size,hidden_size*2)\n",
        "        self.lstm_linear_layer = torch.nn.Linear(1024,256)\n",
        "        \n",
        "        # Block 2 - Conv layer block\n",
        "        self.lstm_layer_2 = torch.nn.LSTM(embedding_matrix.shape[1], hidden_size,\n",
        "                                        num_layers=num_of_layers,\n",
        "                                        batch_first=True, dropout=self.dropout_prob,\n",
        "                                        bidirectional=True)\n",
        "        self.conv_layer_1 = torch.nn.Conv1d(in_channels=512,out_channels=512,kernel_size=3,stride=1)\n",
        "        self.conv_layer_2 = torch.nn.Conv1d(in_channels=512,out_channels=512,kernel_size=5,stride=3)\n",
        "        self.conv_layer_3 = torch.nn.Conv1d(in_channels=512,out_channels=512,kernel_size=11,stride=9)\n",
        "        self.conv_linear_layer_1 = torch.nn.Linear(198,128)\n",
        "        self.conv_linear_layer_2 = torch.nn.Linear(66,128)\n",
        "        self.conv_linear_layer_3 = torch.nn.Linear(22,128)\n",
        "        self.conv_output_linear_layer = torch.nn.Linear(1536,256)\n",
        "        \n",
        "        # Combining both the blocks \n",
        "        self.linear_layer_1 = torch.nn.Linear(256,64)\n",
        "        self.linear_layer_2 = torch.nn.Linear(64,8)\n",
        "        self.linear_layer_3 = torch.nn.Linear(8,1)\n",
        "        self.linear_layer_4 = torch.nn.Linear(328,128)\n",
        "        self.linear_layer_5 = torch.nn.Linear(128,32)\n",
        "        self.linear_layer_6 = torch.nn.Linear(32,8)\n",
        "        self.linear_layer_7 = torch.nn.Linear(8,1)\n",
        "        self.tanh = torch.nn.Tanh()\n",
        "        self.relu = torch.nn.ReLU()\n",
        "        self.leaky_relu = torch.nn.LeakyReLU()\n",
        "        self.dropout_layer = torch.nn.Dropout(p=self.dropout_prob)\n",
        "    \n",
        "    def forward(self,x):\n",
        "        x = self.embeddings_layer(x) #shape of x is 16,200,300\n",
        "        \n",
        "        # Block 1 - Lstm layer block\n",
        "        lstm_output_1, (ht1, ct1) = self.lstm_layer_1(x)\n",
        "        #Shape of lstm_output_1 is 16,200,512 \n",
        "        #Shape of ht1 is 6,16,256 \n",
        "        #Shape of ct2 is 6,16,256\n",
        "        #Shape of ht1[-1] is 16,256\n",
        "        attention_output = self.attention_layer(ht1[-1]) #Shape of is 16,512\n",
        "        attention_output = attention_output.unsqueeze(1) #Shape is 16,1,512\n",
        "        attention_context = lstm_output_1 * attention_output #Shape is 16,200,512\n",
        "        softmax_attention = torch.nn.functional.softmax(attention_context,dim=1) #Shape is 16,200,512\n",
        "        word_context = softmax_attention * lstm_output_1  #Shape is 16,200,512\n",
        "        final_context_words = torch.cat([word_context,lstm_output_1],dim=2)  #Shape is 16,200,1024\n",
        "        block_1_output = self.lstm_linear_layer(final_context_words)  #Shape is 16,200,256\n",
        "        \n",
        "        # Block 2 - Conv layer block\n",
        "        lstm_output_2, (ht2, ct2) = self.lstm_layer_2(x)\n",
        "        #Shape of lstm_output_2 is 16,200,512 \n",
        "        #Shape of ht2 is 6,16,256 \n",
        "        #Shape of ct2 is 6,16,256\n",
        "        #Shape of ht2[-1] is 16,256\n",
        "        lstm_output_2_permuted = lstm_output_2.permute(0,2,1)   #Shape is 16,512,200\n",
        "        conv_output_1 = self.conv_layer_1(lstm_output_2_permuted)       #Shape is 16,512,198\n",
        "        conv_output_2 = self.conv_layer_2(lstm_output_2_permuted)       #Shape is 16,512,66\n",
        "        conv_output_3 = self.conv_layer_3(lstm_output_2_permuted)       #Shape is 16,512,22\n",
        "        conv_output_linear_1 = self.conv_linear_layer_1(conv_output_1)  #Shape is 16,512,128\n",
        "        conv_output_linear_2 = self.conv_linear_layer_2(conv_output_2)  #Shape is 16,512,128\n",
        "        conv_output_linear_3 = self.conv_linear_layer_3(conv_output_3)  #Shape is 16,512,128\n",
        "        conv_output = torch.cat([conv_output_linear_1,conv_output_linear_2,conv_output_linear_3],\n",
        "                                dim=1)  #Shape is 16,1536,128\n",
        "        conv_output = conv_output.permute(0,2,1) #Shape is 16,128,1536\n",
        "        block_2_output = self.conv_output_linear_layer(conv_output)  #Shape is 16,128,256\n",
        "        \n",
        "        # Combining block 1 & 2\n",
        "        concatenated_output = torch.cat([block_1_output,block_2_output],dim=1)  #Shape is 16,328,256\n",
        "        linear_output_1 = self.linear_layer_1(concatenated_output)  #Shape is 16,328,64\n",
        "        linear_output_1 = self.leaky_relu(linear_output_1)\n",
        "        linear_output_1 = self.dropout_layer(linear_output_1)\n",
        "        \n",
        "        linear_output_2 = self.linear_layer_2(linear_output_1)  #Shape is 16,328,8\n",
        "        linear_output_2 = self.leaky_relu(linear_output_2)\n",
        "        linear_output_2 = self.dropout_layer(linear_output_2)\n",
        "        \n",
        "        linear_output_3 = self.linear_layer_3(linear_output_2)  #Shape is 16,328,1\n",
        "        linear_output_3 = self.dropout_layer(linear_output_3)\n",
        "        linear_output_3 = linear_output_3.squeeze(2)            #Shape is 16,328\n",
        "        \n",
        "        linear_output_4 = self.linear_layer_4(linear_output_3)  #Shape is 16,128\n",
        "        linear_output_4 = self.leaky_relu(linear_output_4)\n",
        "        linear_output_4 = self.dropout_layer(linear_output_4)\n",
        "        \n",
        "        linear_output_5 = self.linear_layer_5(linear_output_4)  #Shape is 16,32\n",
        "        linear_output_5 = self.leaky_relu(linear_output_5)        \n",
        "        linear_output_5 = self.dropout_layer(linear_output_5)\n",
        "        \n",
        "        linear_output_6 = self.linear_layer_6(linear_output_5)  #Shape is 16,8\n",
        "        linear_output_6 = self.dropout_layer(linear_output_6)\n",
        "\n",
        "        linear_output_7 = self.linear_layer_7(linear_output_6)  #Shape is 16,1\n",
        "        output = linear_output_7\n",
        "        return output\n",
        "\n",
        "# Average Validation Loss:\n",
        "# Lowest validation loss is  at fold 3"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-c0e916e9ab83>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mReadabilityModel_parallel_LSTM_Attention_and_LSTM_CNN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membedding_matrix\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_of_layers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtraining\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout_prob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'torch' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OBrunmnJi74U"
      },
      "source": [
        "# Mayank\n",
        "\n",
        "class ReadabilityModel_LSTM_1_CNN(torch.nn.Module):\n",
        "    def __init__(self, embedding_matrix, hidden_size, num_of_layers):\n",
        "        super().__init__()\n",
        "        self.dropout_probability = 0.2\n",
        "        self.embeddings_layer = torch.nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n",
        "                                                   padding_idx=0)\n",
        "        self.embeddings_layer.weight = torch.nn.Parameter(torch.tensor(embedding_matrix, \n",
        "                                                                       dtype=torch.float32,\n",
        "                                                                       device=device,CUDA_LAUNCH_BLOCKING=1))\n",
        "        self.embeddings_layer.weight.requires_grad = False\n",
        "        self.lstm_layer = torch.nn.LSTM(300, hidden_size,\n",
        "                                        num_layers=num_of_layers, dropout=self.dropout_probability,\n",
        "                                        batch_first=True, bidirectional=True)\n",
        "        self.conv_layer_1 = torch.nn.Conv1d(in_channels=512,out_channels=512,kernel_size=3,stride=1)\n",
        "        self.linear_layer_1 = torch.nn.Linear(185,128)\n",
        "        self.linear_layer_2 = torch.nn.Linear(128,64)\n",
        "        self.linear_layer_3 = torch.nn.Linear(64,16)\n",
        "        self.linear_layer_4 = torch.nn.Linear(16,1)\n",
        "        self.linear_layer_5 = torch.nn.Linear(512,128)\n",
        "        self.linear_layer_6 = torch.nn.Linear(128,32)\n",
        "        self.linear_layer_7 = torch.nn.Linear(32,5)\n",
        "        self.dropout_layer = torch.nn.Dropout(p=self.dropout_probability)\n",
        "        self.leaky_relu = torch.nn.LeakyReLU()\n",
        "        # self.maxpool_layer = torch.nn.MaxPool1d(3,stride=2)\n",
        "\n",
        "    def forward(self,x):\n",
        "        x = self.embeddings_layer(x) #shape of x is 16,187,300\n",
        "        lstm_output, (ht, ct) = self.lstm_layer(x)\n",
        "        #Shape of lstm_output is 16,187,512\n",
        "        #Shape of ht1 is 6,16,256\n",
        "        #Shape of ct2 is 6,16,256\n",
        "        # print(lstm_output.shape)\n",
        "        lstm_output = lstm_output.permute(0,2,1) #Shape of lstm_output is 16,512,187\n",
        "        conv_output_1 = self.conv_layer_1(lstm_output) #shape of conv_output is 16,512,185\n",
        "        linear_output_1 = self.linear_layer_1(conv_output_1)\n",
        "        linear_output_1 = self.leaky_relu(linear_output_1)\n",
        "        linear_output_1 = self.dropout_layer(linear_output_1)\n",
        "\n",
        "        linear_output_2 = self.linear_layer_2(linear_output_1)\n",
        "        linear_output_2 = self.leaky_relu(linear_output_2)\n",
        "        linear_output_2 = self.dropout_layer(linear_output_2)\n",
        "\n",
        "        linear_output_3 = self.linear_layer_3(linear_output_2)\n",
        "        linear_output_3 = self.leaky_relu(linear_output_3)\n",
        "        linear_output_3 = self.dropout_layer(linear_output_3)\n",
        "\n",
        "        linear_output_4 = self.linear_layer_4(linear_output_3)\n",
        "        linear_output_4 = self.leaky_relu(linear_output_4)\n",
        "        linear_output_4 = self.dropout_layer(linear_output_4)\n",
        "        linear_output_4 = linear_output_4.squeeze(2)\n",
        "\n",
        "        linear_output_5 = self.linear_layer_5(linear_output_4)\n",
        "        linear_output_5 = self.leaky_relu(linear_output_5)\n",
        "        linear_output_5 = self.dropout_layer(linear_output_5)\n",
        "\n",
        "        linear_output_6 = self.linear_layer_6(linear_output_5)\n",
        "        linear_output_6 = self.leaky_relu(linear_output_6)\n",
        "        linear_output_6 = self.dropout_layer(linear_output_6)\n",
        "\n",
        "        linear_output_7 = self.linear_layer_7(linear_output_6)\n",
        "        output = linear_output_7\n",
        "        return output"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6Ci_ao8XIVWM"
      },
      "source": [
        "#Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-06-17T13:07:07.025597Z",
          "iopub.execute_input": "2021-06-17T13:07:07.025948Z",
          "iopub.status.idle": "2021-06-17T17:01:04.936288Z",
          "shell.execute_reply.started": "2021-06-17T13:07:07.025916Z",
          "shell.execute_reply": "2021-06-17T17:01:04.935397Z"
        },
        "trusted": true,
        "id": "H6frPLDYC0KU"
      },
      "source": [
        "def train(train_dataset,valid_dataset,epochs,learning_rate,train_batch_size,valid_batch_size,embedding_matrix,hidden_size, num_of_layers):\n",
        "  loss_function = torch.nn.CrossEntropyLoss()\n",
        "\n",
        "  trainloader = torch.utils.data.DataLoader(train_dataset,batch_size=train_batch_size,\n",
        "                                            drop_last=True)\n",
        "  validloader = torch.utils.data.DataLoader(valid_dataset,batch_size=valid_batch_size,\n",
        "                                            drop_last=True)\n",
        "\n",
        "  model = ReadabilityModel_parallel_LSTM_Attention_and_CNN(embedding_matrix, hidden_size, num_of_layers)\n",
        "  model = model.to(device)\n",
        "  train_loss_list,valid_loss_list = [],[]\n",
        "  train_accuracy_list,valid_accuracy_list = [],[]\n",
        "  for epoch in range(0, epochs):\n",
        "      epoch_loss = 0.0\n",
        "      model.train()\n",
        "      train_correct = 0\n",
        "      # learning_rate = 0.001/(epoch+1)\n",
        "      optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
        "      for i, data in enumerate(trainloader, 0):\n",
        "          inputs, targets = data\n",
        "          optimizer.zero_grad()\n",
        "          targets = torch.tensor(targets,dtype=torch.long,device=device)\n",
        "          outputs = model(inputs)\n",
        "          output_ids = torch.argmax(outputs, dim=1)\n",
        "\n",
        "          loss = loss_function(outputs, targets)\n",
        "          epoch_loss += loss.item()\n",
        "\n",
        "          loss.backward()\n",
        "          optimizer.step()\n",
        "          train_correct += (output_ids == targets).float().sum()\n",
        "\n",
        "      train_accuracy = 100 * train_correct / len(train_dataset)\n",
        "      epoch_loss /= len(trainloader) / train_batch_size \n",
        "      train_loss_list.append(epoch_loss)\n",
        "      train_accuracy_list.append(train_accuracy)\n",
        "\n",
        "      with torch.no_grad():\n",
        "        validation_loss = 0.0\n",
        "        valid_correct = 0\n",
        "        for i, data in enumerate(validloader, 0):\n",
        "            model.eval()\n",
        "            inputs, targets = data\n",
        "            targets = torch.tensor(targets,dtype=torch.long,device=device)\n",
        "            outputs = model(inputs)\n",
        "            output_ids = torch.argmax(outputs, dim=1)\n",
        "            # outputs = outputs.squeeze(1)\n",
        "            loss = loss_function(outputs, targets)\n",
        "            validation_loss += loss.item()\n",
        "            valid_correct += (output_ids == targets).float().sum()\n",
        "\n",
        "      valid_accuracy = 100 * valid_correct / len(valid_dataset)  \n",
        "      validation_loss /= len(validloader) / valid_batch_size \n",
        "      valid_loss_list.append(validation_loss)\n",
        "      valid_accuracy_list.append(valid_accuracy)\n",
        "      print(f'Epoch:{epoch}, Training Loss:{epoch_loss} Validation Loss: {validation_loss} \\n Training accuracy:{train_accuracy} Validation Accuracy: {valid_accuracy} ')\n",
        "\n",
        "      torch.save(model,'/content/drive/MyDrive/Readability_Research_Paper/models/parallel_lstm_3_cnn/lstm_{}.pkl'.format(epoch))    \n",
        "      print('--------------------------------')\n",
        "\n",
        "  return train_loss_list,valid_loss_list,train_accuracy_list,valid_accuracy_list,model"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xvueADn3iPQg"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "split_train_df, split_val_df = train_test_split(train_dataset,test_size=0.1,stratify=train_dataset['readability'])"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xFBKon58cgZn"
      },
      "source": [
        "train_dataset_torch = WeebitDataset(split_train_df)\n",
        "valid_dataset_torch = WeebitDataset(split_val_df)\n",
        "EMBEDDING_SIZE = 300 \n",
        "hidden_size = 256\n",
        "num_of_layers = 3\n",
        "train_batch_size = 16\n",
        "valid_batch_size = 1\n",
        "epochs = 100\n",
        "learning_rate = 0.00005"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6Q5ke-Vg7K1m",
        "outputId": "15e23aef-c8db-4ff7-c14d-db77f9b98f82"
      },
      "source": [
        "a = torch.tensor([1,2],dtype=torch.float64,device=device)\n",
        "print(a.is_cuda)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "True\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d8b6YirPC0KV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "890d2053-3a8a-49f7-a0f5-2513f76b1473"
      },
      "source": [
        "train_loss_list,valid_loss_list,train_accuracy_list,valid_accuracy_list,model = train(train_dataset_torch,valid_dataset_torch,epochs,learning_rate,\n",
        "                                                                                      train_batch_size,valid_batch_size,\n",
        "                                                                                      embedding_matrix,hidden_size,num_of_layers)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:44: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch:0, Training Loss:25.861067063467843 Validation Loss: 1.6143344130516053 \n",
            " Training accuracy:21.377777099609375 Validation Accuracy: 20.0 \n",
            "--------------------------------\n",
            "Epoch:1, Training Loss:25.809365449632917 Validation Loss: 1.6125546746253967 \n",
            " Training accuracy:20.22222328186035 Validation Accuracy: 20.0 \n",
            "--------------------------------\n",
            "Epoch:2, Training Loss:25.695242077963695 Validation Loss: 1.5499661741256714 \n",
            " Training accuracy:21.511112213134766 Validation Accuracy: 27.600000381469727 \n",
            "--------------------------------\n",
            "Epoch:3, Training Loss:24.077482782091412 Validation Loss: 1.4339391951560974 \n",
            " Training accuracy:32.488887786865234 Validation Accuracy: 39.20000076293945 \n",
            "--------------------------------\n",
            "Epoch:4, Training Loss:23.366086537497385 Validation Loss: 1.3819225776195525 \n",
            " Training accuracy:36.79999923706055 Validation Accuracy: 42.000003814697266 \n",
            "--------------------------------\n",
            "Epoch:5, Training Loss:21.90046532494681 Validation Loss: 1.2602731654644013 \n",
            " Training accuracy:41.55555725097656 Validation Accuracy: 51.60000228881836 \n",
            "--------------------------------\n",
            "Epoch:6, Training Loss:20.590545674732752 Validation Loss: 1.1826239477396012 \n",
            " Training accuracy:42.53333282470703 Validation Accuracy: 49.60000228881836 \n",
            "--------------------------------\n",
            "Epoch:7, Training Loss:19.47830888203212 Validation Loss: 1.1974242911338806 \n",
            " Training accuracy:45.511112213134766 Validation Accuracy: 47.20000076293945 \n",
            "--------------------------------\n",
            "Epoch:8, Training Loss:19.029573590414866 Validation Loss: 1.1495227336883544 \n",
            " Training accuracy:45.68888854980469 Validation Accuracy: 47.20000076293945 \n",
            "--------------------------------\n",
            "Epoch:9, Training Loss:18.042862912586756 Validation Loss: 1.0634606951475143 \n",
            " Training accuracy:48.79999923706055 Validation Accuracy: 52.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:10, Training Loss:17.91157489504133 Validation Loss: 1.0669976046085359 \n",
            " Training accuracy:50.35555648803711 Validation Accuracy: 52.000003814697266 \n",
            "--------------------------------\n",
            "Epoch:11, Training Loss:17.446864891052247 Validation Loss: 1.0410028920173644 \n",
            " Training accuracy:49.82222366333008 Validation Accuracy: 52.400001525878906 \n",
            "--------------------------------\n",
            "Epoch:12, Training Loss:17.230666862215315 Validation Loss: 1.0243822071552278 \n",
            " Training accuracy:51.02222442626953 Validation Accuracy: 51.60000228881836 \n",
            "--------------------------------\n",
            "Epoch:13, Training Loss:17.043502487455097 Validation Loss: 1.0222620223164558 \n",
            " Training accuracy:50.88888931274414 Validation Accuracy: 52.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:14, Training Loss:16.823375034332276 Validation Loss: 1.0196995176076888 \n",
            " Training accuracy:51.42222213745117 Validation Accuracy: 52.000003814697266 \n",
            "--------------------------------\n",
            "Epoch:15, Training Loss:16.036958926064628 Validation Loss: 0.9920473617613316 \n",
            " Training accuracy:53.28889083862305 Validation Accuracy: 52.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:16, Training Loss:16.693333189828056 Validation Loss: 1.0006006741821767 \n",
            " Training accuracy:50.400001525878906 Validation Accuracy: 52.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:17, Training Loss:16.0150564125606 Validation Loss: 0.9699277873635292 \n",
            " Training accuracy:54.31111145019531 Validation Accuracy: 54.400001525878906 \n",
            "--------------------------------\n",
            "Epoch:18, Training Loss:15.766254990441459 Validation Loss: 0.9544773666858674 \n",
            " Training accuracy:53.60000228881836 Validation Accuracy: 54.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:19, Training Loss:15.677678094591414 Validation Loss: 0.8979456941783428 \n",
            " Training accuracy:54.4444465637207 Validation Accuracy: 58.000003814697266 \n",
            "--------------------------------\n",
            "Epoch:20, Training Loss:15.090288468769618 Validation Loss: 0.8851820789724588 \n",
            " Training accuracy:55.20000076293945 Validation Accuracy: 62.400001525878906 \n",
            "--------------------------------\n",
            "Epoch:21, Training Loss:14.562828608921595 Validation Loss: 0.85484049089998 \n",
            " Training accuracy:58.53333282470703 Validation Accuracy: 63.20000457763672 \n",
            "--------------------------------\n",
            "Epoch:22, Training Loss:13.840143878119333 Validation Loss: 0.824968209080398 \n",
            " Training accuracy:60.31111145019531 Validation Accuracy: 65.60000610351562 \n",
            "--------------------------------\n",
            "Epoch:23, Training Loss:13.451260287421091 Validation Loss: 0.8075560713484883 \n",
            " Training accuracy:62.755558013916016 Validation Accuracy: 68.0 \n",
            "--------------------------------\n",
            "Epoch:24, Training Loss:13.58128456388201 Validation Loss: 0.8162389501035213 \n",
            " Training accuracy:62.4444465637207 Validation Accuracy: 68.4000015258789 \n",
            "--------------------------------\n",
            "Epoch:25, Training Loss:13.142964332444327 Validation Loss: 0.8774218486342579 \n",
            " Training accuracy:62.57777786254883 Validation Accuracy: 66.0 \n",
            "--------------------------------\n",
            "Epoch:26, Training Loss:12.633579472133091 Validation Loss: 0.8142010645978153 \n",
            " Training accuracy:64.62222290039062 Validation Accuracy: 66.4000015258789 \n",
            "--------------------------------\n",
            "Epoch:27, Training Loss:12.43641082218715 Validation Loss: 0.8594048476712778 \n",
            " Training accuracy:64.97777557373047 Validation Accuracy: 66.0 \n",
            "--------------------------------\n",
            "Epoch:28, Training Loss:12.784670223508563 Validation Loss: 0.8314157950598746 \n",
            " Training accuracy:64.71111297607422 Validation Accuracy: 63.60000228881836 \n",
            "--------------------------------\n",
            "Epoch:29, Training Loss:12.277086465699332 Validation Loss: 0.8789274590978166 \n",
            " Training accuracy:65.46666717529297 Validation Accuracy: 64.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:30, Training Loss:12.164156941005162 Validation Loss: 0.8230909850371536 \n",
            " Training accuracy:67.11111450195312 Validation Accuracy: 64.4000015258789 \n",
            "--------------------------------\n",
            "Epoch:31, Training Loss:12.004704236984253 Validation Loss: 0.836263692237204 \n",
            " Training accuracy:66.22222137451172 Validation Accuracy: 65.60000610351562 \n",
            "--------------------------------\n",
            "Epoch:32, Training Loss:11.53435001713889 Validation Loss: 0.8342009423280542 \n",
            " Training accuracy:67.55555725097656 Validation Accuracy: 66.0 \n",
            "--------------------------------\n",
            "Epoch:33, Training Loss:11.495116526739938 Validation Loss: 0.8192107312086154 \n",
            " Training accuracy:68.13333129882812 Validation Accuracy: 66.4000015258789 \n",
            "--------------------------------\n",
            "Epoch:34, Training Loss:11.358687652860368 Validation Loss: 0.8533370275939814 \n",
            " Training accuracy:68.17778015136719 Validation Accuracy: 65.60000610351562 \n",
            "--------------------------------\n",
            "Epoch:35, Training Loss:11.411249436650957 Validation Loss: 0.8275148359199811 \n",
            " Training accuracy:68.53333282470703 Validation Accuracy: 64.4000015258789 \n",
            "--------------------------------\n",
            "Epoch:36, Training Loss:10.758183785847255 Validation Loss: 0.8236243940207132 \n",
            " Training accuracy:69.5999984741211 Validation Accuracy: 65.60000610351562 \n",
            "--------------------------------\n",
            "Epoch:37, Training Loss:10.988162650380815 Validation Loss: 0.8699711397213395 \n",
            " Training accuracy:69.55555725097656 Validation Accuracy: 64.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:38, Training Loss:10.489442012991224 Validation Loss: 0.8543898765659287 \n",
            " Training accuracy:70.66666412353516 Validation Accuracy: 64.0 \n",
            "--------------------------------\n",
            "Epoch:39, Training Loss:10.199207055568696 Validation Loss: 0.8838834916194355 \n",
            " Training accuracy:72.35555267333984 Validation Accuracy: 62.400001525878906 \n",
            "--------------------------------\n",
            "Epoch:40, Training Loss:10.186512703554971 Validation Loss: 0.8677943698936106 \n",
            " Training accuracy:73.11111450195312 Validation Accuracy: 64.4000015258789 \n",
            "--------------------------------\n",
            "Epoch:41, Training Loss:10.136379817553928 Validation Loss: 1.025453449643479 \n",
            " Training accuracy:72.0888900756836 Validation Accuracy: 59.60000228881836 \n",
            "--------------------------------\n",
            "Epoch:42, Training Loss:10.06259845835822 Validation Loss: 0.8743062568787263 \n",
            " Training accuracy:73.82221984863281 Validation Accuracy: 62.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:43, Training Loss:9.741119691303798 Validation Loss: 0.8736851686304553 \n",
            " Training accuracy:73.37777709960938 Validation Accuracy: 64.4000015258789 \n",
            "--------------------------------\n",
            "Epoch:44, Training Loss:9.731287399360111 Validation Loss: 0.8669175968750124 \n",
            " Training accuracy:73.55555725097656 Validation Accuracy: 66.0 \n",
            "--------------------------------\n",
            "Epoch:45, Training Loss:9.666963716915676 Validation Loss: 0.8748322382003239 \n",
            " Training accuracy:73.02222442626953 Validation Accuracy: 64.4000015258789 \n",
            "--------------------------------\n",
            "Epoch:46, Training Loss:9.366945511954171 Validation Loss: 0.9244871735527231 \n",
            " Training accuracy:75.42222595214844 Validation Accuracy: 64.0 \n",
            "--------------------------------\n",
            "Epoch:47, Training Loss:9.367843036992209 Validation Loss: 0.8625559608098361 \n",
            " Training accuracy:75.64444732666016 Validation Accuracy: 65.60000610351562 \n",
            "--------------------------------\n",
            "Epoch:48, Training Loss:8.8015927195549 Validation Loss: 0.9714574117449081 \n",
            " Training accuracy:75.5999984741211 Validation Accuracy: 60.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:49, Training Loss:8.537571177312307 Validation Loss: 0.9778887685710058 \n",
            " Training accuracy:77.64444732666016 Validation Accuracy: 60.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:50, Training Loss:8.510106320040567 Validation Loss: 0.961341153356421 \n",
            " Training accuracy:77.5111083984375 Validation Accuracy: 63.20000457763672 \n",
            "--------------------------------\n",
            "Epoch:51, Training Loss:8.308749662126814 Validation Loss: 1.0072018144810697 \n",
            " Training accuracy:78.0 Validation Accuracy: 63.20000457763672 \n",
            "--------------------------------\n",
            "Epoch:52, Training Loss:8.374758061340877 Validation Loss: 0.9559108947302459 \n",
            " Training accuracy:77.42222595214844 Validation Accuracy: 65.20000457763672 \n",
            "--------------------------------\n",
            "Epoch:53, Training Loss:8.089451694488526 Validation Loss: 0.9731253381442797 \n",
            " Training accuracy:78.44444274902344 Validation Accuracy: 64.0 \n",
            "--------------------------------\n",
            "Epoch:54, Training Loss:7.707306600468499 Validation Loss: 1.0521706372259165 \n",
            " Training accuracy:79.95555877685547 Validation Accuracy: 64.4000015258789 \n",
            "--------------------------------\n",
            "Epoch:55, Training Loss:7.564564695528575 Validation Loss: 0.9850346967311792 \n",
            " Training accuracy:80.35555267333984 Validation Accuracy: 65.60000610351562 \n",
            "--------------------------------\n",
            "Epoch:56, Training Loss:7.353740082468305 Validation Loss: 1.0533940380365765 \n",
            " Training accuracy:80.71111297607422 Validation Accuracy: 61.20000457763672 \n",
            "--------------------------------\n",
            "Epoch:57, Training Loss:7.69600037080901 Validation Loss: 0.9561642807687214 \n",
            " Training accuracy:81.02222442626953 Validation Accuracy: 64.0 \n",
            "--------------------------------\n",
            "Epoch:58, Training Loss:7.24559565441949 Validation Loss: 1.0212009558876691 \n",
            " Training accuracy:81.20000457763672 Validation Accuracy: 64.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:59, Training Loss:7.225101698722158 Validation Loss: 1.1557845400564706 \n",
            " Training accuracy:82.35555267333984 Validation Accuracy: 63.20000457763672 \n",
            "--------------------------------\n",
            "Epoch:60, Training Loss:7.274184410061155 Validation Loss: 1.0864284194757718 \n",
            " Training accuracy:81.02222442626953 Validation Accuracy: 64.0 \n",
            "--------------------------------\n",
            "Epoch:61, Training Loss:6.984910555396762 Validation Loss: 1.3079030162078114 \n",
            " Training accuracy:82.0888900756836 Validation Accuracy: 60.000003814697266 \n",
            "--------------------------------\n",
            "Epoch:62, Training Loss:6.581573589359011 Validation Loss: 1.162047624252844 \n",
            " Training accuracy:83.68888854980469 Validation Accuracy: 65.20000457763672 \n",
            "--------------------------------\n",
            "Epoch:63, Training Loss:6.5637863806315835 Validation Loss: 1.2263442800545088 \n",
            " Training accuracy:83.68888854980469 Validation Accuracy: 61.60000228881836 \n",
            "--------------------------------\n",
            "Epoch:64, Training Loss:6.1999763312084335 Validation Loss: 1.3468360847246899 \n",
            " Training accuracy:83.42222595214844 Validation Accuracy: 60.000003814697266 \n",
            "--------------------------------\n",
            "Epoch:65, Training Loss:6.226302005563464 Validation Loss: 1.3009033303544126 \n",
            " Training accuracy:85.02222442626953 Validation Accuracy: 59.20000457763672 \n",
            "--------------------------------\n",
            "Epoch:66, Training Loss:6.125674857199192 Validation Loss: 1.230809814171494 \n",
            " Training accuracy:84.62222290039062 Validation Accuracy: 61.20000457763672 \n",
            "--------------------------------\n",
            "Epoch:67, Training Loss:6.197691913161959 Validation Loss: 1.214994110096021 \n",
            " Training accuracy:84.75555419921875 Validation Accuracy: 61.60000228881836 \n",
            "--------------------------------\n",
            "Epoch:68, Training Loss:5.83728269977229 Validation Loss: 1.2619149035421056 \n",
            " Training accuracy:86.04444885253906 Validation Accuracy: 61.60000228881836 \n",
            "--------------------------------\n",
            "Epoch:69, Training Loss:5.5452537674989015 Validation Loss: 1.3060310725829298 \n",
            " Training accuracy:86.62222290039062 Validation Accuracy: 60.400001525878906 \n",
            "--------------------------------\n",
            "Epoch:70, Training Loss:5.689785472410065 Validation Loss: 1.4966691708930298 \n",
            " Training accuracy:85.82221984863281 Validation Accuracy: 58.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:71, Training Loss:5.66418342760631 Validation Loss: 1.3966313952352976 \n",
            " Training accuracy:86.22222137451172 Validation Accuracy: 61.20000457763672 \n",
            "--------------------------------\n",
            "Epoch:72, Training Loss:5.203293220911707 Validation Loss: 1.4398526966936969 \n",
            " Training accuracy:87.5111083984375 Validation Accuracy: 60.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:73, Training Loss:5.308439994922706 Validation Loss: 1.42068544600858 \n",
            " Training accuracy:86.8888931274414 Validation Accuracy: 62.000003814697266 \n",
            "--------------------------------\n",
            "Epoch:74, Training Loss:5.318198334319251 Validation Loss: 1.4388187855300325 \n",
            " Training accuracy:87.77777862548828 Validation Accuracy: 60.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:75, Training Loss:5.2699745787041525 Validation Loss: 1.575309180767781 \n",
            " Training accuracy:87.42222595214844 Validation Accuracy: 60.000003814697266 \n",
            "--------------------------------\n",
            "Epoch:76, Training Loss:4.993236423390252 Validation Loss: 1.4494580074553924 \n",
            " Training accuracy:88.66666412353516 Validation Accuracy: 62.400001525878906 \n",
            "--------------------------------\n",
            "Epoch:77, Training Loss:5.1300992820944105 Validation Loss: 1.6821864937016426 \n",
            " Training accuracy:87.46666717529297 Validation Accuracy: 58.000003814697266 \n",
            "--------------------------------\n",
            "Epoch:78, Training Loss:4.443665361830166 Validation Loss: 1.458817943247329 \n",
            " Training accuracy:89.46666717529297 Validation Accuracy: 64.0 \n",
            "--------------------------------\n",
            "Epoch:79, Training Loss:4.277246361119406 Validation Loss: 1.678703522937066 \n",
            " Training accuracy:89.73333740234375 Validation Accuracy: 61.20000457763672 \n",
            "--------------------------------\n",
            "Epoch:80, Training Loss:4.413033922016621 Validation Loss: 1.6313582131575046 \n",
            " Training accuracy:89.5111083984375 Validation Accuracy: 60.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:81, Training Loss:4.642276540824345 Validation Loss: 1.6996006030701314 \n",
            " Training accuracy:89.42222595214844 Validation Accuracy: 56.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:82, Training Loss:4.73375775792769 Validation Loss: 1.8657214315107526 \n",
            " Training accuracy:89.37777709960938 Validation Accuracy: 58.000003814697266 \n",
            "--------------------------------\n",
            "Epoch:83, Training Loss:4.0309918912393705 Validation Loss: 1.9975949490107696 \n",
            " Training accuracy:90.04444885253906 Validation Accuracy: 58.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:84, Training Loss:4.335865466935294 Validation Loss: 1.633495030252933 \n",
            " Training accuracy:90.62222290039062 Validation Accuracy: 60.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:85, Training Loss:4.138088446216924 Validation Loss: 2.1595364073802554 \n",
            " Training accuracy:90.5777816772461 Validation Accuracy: 56.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:86, Training Loss:4.100215220238481 Validation Loss: 1.6200814108169803 \n",
            " Training accuracy:90.80000305175781 Validation Accuracy: 60.400001525878906 \n",
            "--------------------------------\n",
            "Epoch:87, Training Loss:4.0441308178007604 Validation Loss: 1.9896843911053588 \n",
            " Training accuracy:90.8888931274414 Validation Accuracy: 57.20000457763672 \n",
            "--------------------------------\n",
            "Epoch:88, Training Loss:3.8583957654024874 Validation Loss: 1.8960113945061698 \n",
            " Training accuracy:91.11111450195312 Validation Accuracy: 56.400001525878906 \n",
            "--------------------------------\n",
            "Epoch:89, Training Loss:3.8155246821365187 Validation Loss: 1.9335827273424595 \n",
            " Training accuracy:91.15555572509766 Validation Accuracy: 60.000003814697266 \n",
            "--------------------------------\n",
            "Epoch:90, Training Loss:3.8791878038219045 Validation Loss: 1.8510554631264493 \n",
            " Training accuracy:91.06666564941406 Validation Accuracy: 59.60000228881836 \n",
            "--------------------------------\n",
            "Epoch:91, Training Loss:3.349091387646539 Validation Loss: 1.9147145077869363 \n",
            " Training accuracy:92.13333129882812 Validation Accuracy: 60.400001525878906 \n",
            "--------------------------------\n",
            "Epoch:92, Training Loss:3.753651376547558 Validation Loss: 1.967550663264609 \n",
            " Training accuracy:91.37777709960938 Validation Accuracy: 59.20000457763672 \n",
            "--------------------------------\n",
            "Epoch:93, Training Loss:4.4520446463355 Validation Loss: 1.9386124333726857 \n",
            " Training accuracy:91.24444580078125 Validation Accuracy: 60.400001525878906 \n",
            "--------------------------------\n",
            "Epoch:94, Training Loss:4.138849295542708 Validation Loss: 1.8927235392542268 \n",
            " Training accuracy:90.97777557373047 Validation Accuracy: 59.20000457763672 \n",
            "--------------------------------\n",
            "Epoch:95, Training Loss:3.118076008132526 Validation Loss: 2.2257931405901115 \n",
            " Training accuracy:92.8888931274414 Validation Accuracy: 56.80000305175781 \n",
            "--------------------------------\n",
            "Epoch:96, Training Loss:3.575740859657526 Validation Loss: 2.063222309639749 \n",
            " Training accuracy:92.35556030273438 Validation Accuracy: 59.60000228881836 \n",
            "--------------------------------\n",
            "Epoch:97, Training Loss:3.3007121537412916 Validation Loss: 1.7999837995419747 \n",
            " Training accuracy:92.75555419921875 Validation Accuracy: 64.4000015258789 \n",
            "--------------------------------\n",
            "Epoch:98, Training Loss:3.010548428233181 Validation Loss: 1.9336894085401708 \n",
            " Training accuracy:93.33333587646484 Validation Accuracy: 61.60000228881836 \n",
            "--------------------------------\n",
            "Epoch:99, Training Loss:3.1420257116002697 Validation Loss: 2.04562573748417 \n",
            " Training accuracy:93.28888702392578 Validation Accuracy: 62.000003814697266 \n",
            "--------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "JC_nrAEPd0aQ",
        "outputId": "f80fea80-95ad-4f96-ede9-6ebfe00319af"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.plot(train_loss_list,'r',valid_loss_list , 'b')\n",
        "plt.legend([\"Train Loss\", \"Validation Loss\"], loc =\"upper right\")\n",
        "plt.show()"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5zN1frA8c+au2tmGJVbM8ptMBfkGpE4hahQ6EKUk18lFalzji5KpVTSOV2lEImUFFI5idLBkGuoMBh3Q+Nubuv3x7PHDGbMdc939t7P+/Xar5n93bfnO9/Zz157fdd6lrHWopRSyvP4OR2AUkqpwtEErpRSHkoTuFJKeShN4Eop5aE0gSullIcKKMkXq1Klio2IiCjJl1RKKY+3atWqQ9ba8PO3l2gCj4iIID4+viRfUimlPJ4xZkdO27ULRSmlPJQmcKWU8lCawJVSykOVaB+4UqpkpKamkpiYyOnTp50ORRVASEgINWrUIDAwMF/31wSulBdKTEykQoUKREREYIxxOhyVD9ZakpKSSExMJDIyMl+P0S4UpbzQ6dOnqVy5siZvD2KMoXLlygX61qQJXCkvpcnb8xT0mHlGF8rXX8P69VC7tlzq1oVLLnE6KqWUcpRnJPBvvoH//Cfrup8ftGoFXbtCr15Qp45zsSmlLpCUlETHjh0B2LdvH/7+/oSHy0TCFStWEBQUlOtj4+PjmTJlChMmTMj362VOEqxSpUrRAvcwpiQXdGjWrJkt9EzM48dh+3bYtg1WrYJ582D1aihXDhISwMcOnFIXs2nTJho0aOB0GAA888wzlC9fnuHDh5/dlpaWRkBA8bUfvSmB53TsjDGrrLXNzr+v5/SBly8PjRtDjx4werQk8e+/hxMnYOlSp6NTSuVhwIAB3H///bRo0YLHH3+cFStW0KpVK+Li4mjdujVbtmwBYPHixXTr1g2Q5D9w4EDat29P7dq1C9QqT0hI4LrrriM6OpqOHTuyc+dOAGbNmkWjRo2IiYmhXbt2AGzcuJHmzZsTGxtLdHQ0f/zxRzHvvXt4RhdKbtq2hZAQWLIEbrnF6WiUKp2GDYM1a4r3OWNjYfz4Aj8sMTGRZcuW4e/vz9GjR1m6dCkBAQF8//33/OMf/2D27NkXPGbz5s388MMPHDt2jHr16jFkyJB8jZN+6KGH6N+/P/3792fSpEkMHTqUOXPmMHr0aBYuXEj16tX566+/AHjnnXd4+OGHueOOO0hJSSE9Pb3A++aEPFvgxpiaxpgfjDG/GWM2GmMedm1/xhiz2xizxnXp4v5wzxMUBC1bagtcKQ/Ru3dv/P39AUhOTqZ37940atSIRx55hI0bN+b4mK5duxIcHEyVKlWoWrUq+/fvz9dr/fLLL/Tr1w+Au+66i59++gmANm3aMGDAAN5///2zibpVq1a88MILjB07lh07dlCmTJmi7mqJyE8LPA14zFq72hhTAVhljPnOddvr1tpx7gsvH9q1g+efh6NHoWJFR0NRqlQqREvZXcqVK3f291GjRtGhQwe++OILEhISaN++fY6PCQ4OPvu7v78/aWlpRYrhnXfeYfny5cybN4+mTZuyatUq+vXrR4sWLZg3bx5dunTh3Xff5brrrivS65SEPFvg1tq91trVrt+PAZuA6u4OLN/atYOMDFi2zOlIlFIFkJycTPXqkko++uijYn/+1q1bM2PGDACmTZtG27ZtAdi6dSstWrRg9OjRhIeHs2vXLrZt20bt2rUZOnQoPXr0YN26dcUejzsU6CSmMSYCiAOWuzY9aIxZZ4yZZIwJzeUxg40x8caY+IMHDxYp2By1bAkBAdqNopSHefzxx3nyySeJi4srcqsaIDo6mho1alCjRg0effRR3nzzTT788EOio6OZOnUqb7zxBgAjRoygcePGNGrUiNatWxMTE8PMmTNp1KgRsbGxbNiwgbvvvrvI8ZSEfA8jNMaUB34ExlhrPzfGXAocAizwHHC5tXbgxZ6jSMMIL6ZlSwgM1CSulEtpGkaoCqbYhxEaYwKB2cA0a+3nANba/dbadGttBvA+0LzIkRdWu3awYgVo5TWllA/JzygUA3wAbLLWvpZt++XZ7nYLsKH4w8untm0hJUWSuFJK+Yj8jEJpA9wFrDfGZA4m/QfQ1xgTi3ShJAB/d0uE+XHNNWCMjAd3DcxXSilvl2cCt9b+BORUImt+8YdTSKGhMktzyRKnI1FKqRLjOVPp89K2rQwlLIaz2Uop5Qm8J4F36iR1UWbOdDoSpZQqEd6TwG+6CWJi4J//hDNnnI5GKZ/WoUMHFi5ceM628ePHM2TIkFwf0759ezKHGXfp0uVsnZLsnnnmGcaNu/jk7zlz5vDbb7+dvf7UU0/x/fffFyT8HGUvslVaeE8C9/ODV16R0rJvveV0NEr5tL59+56dBZlpxowZ9O3bN1+Pnz9/PpUqVSrUa5+fwEePHs31119fqOcq7bwngYN0o3TuDM89B0eOOB2NUj6rV69ezJs3j5SUFEBKu+7Zs4e2bdsyZMgQmjVrRsOGDXn66adzfHxERASHDh0CYMyYMdStW5drrrnmbMlZgPfff5+rr76amJgYevbsycmTJ1m2bBlz585lxIgRxMbGsnXrVgYMGMBnn30GwKJFi4iLi6Nx48YMHDiQM65v6xERETz99NM0adKExo0bs3nz5nzv6yeffHJ2ZufIkSMBSE9PZ8CAATRq1IjGjRvz+uuvAzBhwgSioqKIjo6mT58+BfyrXsizy8nm5OWXIS4OXnxRflfKxzlRTTYsLIzmzZuzYMECevTowYwZM7jtttswxjBmzBjCwsJIT0+nY8eOrFu3jujo6ByfZ9WqVcyYMYM1a9aQlpZGkyZNaNq0KQC33nor9913HwD/+te/+OCDD3jooYfo3r073bp1o1evXuc81+nTpxkwYACLFi2ibt263H333bz99tsMGzYMgCpVqrB69Wreeustxo0bx8SJE/P8O+zZs4eRI0eyatUqQkND6dy5M3PmzKFmzZrs3r2bDRtkekxmd9BLL73E9u3bCQ4OzrGLqKC8qwUO0g9+990wYQLs2eN0NEr5rOzdKNm7T2bOnEmTJk2Ii4tj48aN53R3nG/p0qXccsstlC1blooVK9K9e/ezt23YsIG2bdvSuHFjpk2blms52kxbtmwhMjKSunXrAtC/f3+WZBt6fOuttwLQtGlTEhIS8rWPK1eupH379oSHhxMQEMAdd9zBkiVLqF27Ntu2beOhhx7im2++oaKrUmp0dDR33HEHH3/8cbGsSOR9LXCAESNg8mSYPx/uvdfpaJRylFPVZHv06MEjjzzC6tWrOXnyJE2bNmX79u2MGzeOlStXEhoayoABAzhdyBIYAwYMYM6cOcTExPDRRx+xePHiIsWbWba2OErWhoaGsnbtWhYuXMg777zDzJkzmTRpEvPmzWPJkiV89dVXjBkzhvXr1xcpkXtfCxwgKgqqVZMl15RSjihfvjwdOnRg4MCBZ1vfR48epVy5clxyySXs37+fBQsWXPQ52rVrx5w5czh16hTHjh3jq6++OnvbsWPHuPzyy0lNTWXatGlnt1eoUIFjx45d8Fz16tUjISGBP//8E4CpU6dy7bXXFmkfmzdvzo8//sihQ4dIT0/nk08+4dprr+XQoUNkZGTQs2dPnn/+eVavXk1GRga7du2iQ4cOjB07luTkZI4fP16k1/fOFrgxcP310gLPyJARKkqpEte3b19uueWWs10pMTExxMXFUb9+fWrWrEmbNm0u+vgmTZpw++23ExMTQ9WqVbn66qvP3vbcc8/RokULwsPDadGixdmk3adPH+677z4mTJhw9uQlQEhICB9++CG9e/cmLS2Nq6++mvvvv79A+7No0SJq1Khx9vqsWbN46aWX6NChA9ZaunbtSo8ePVi7di333HMPGRkZALz44oukp6dz5513kpycjLWWoUOHFnqkTSbPWZW+oKZOlb7wX3+VMy5K+RAtJ+u5vHNV+oLq2FF+fvfdxe+nlFIeynsTeLVq0heu/eBKKS/lvQkcpB986VJd6EH5pJLsHlXFo6DHzPsT+KlT8MsvTkeiVIkKCQkhKSlJk7gHsdaSlJRESEhIvh/jnaNQMl17Lfj7SzdKhw5OR6NUialRowaJiYm4ZSFx5TYhISHnjHLJi3cn8IoVoUULSeBjxjgdjVIlJjAwkMjISKfDUG7m3V0oIN0o8fFa3Eop5XV8I4FnZMCiRU5HopRSxcr7E3irVlCpEsyb53QkSilVrLw/gQcEwI03SgJ3TWtVSilv4P0JHKBbNzh4EFaudDoSpZQqNr6RwG+4QYYTfv2105EopVSx8Y0EHhYGbdpoAldKeRXfSOAg3Shr1sCuXU5HopRSxcK3EjjoaBSllNfwnQRevz7Urq3dKEopr+E7CdwYuOkmmdBz8qTT0SilVJH5TgIH6UY5fVprhCulvIJvJfB27WREyqefOh2JUkoVWZ4J3BhT0xjzgzHmN2PMRmPMw67tYcaY74wxf7h+hro/3CIKCoJeveDLL+HECaejUUqpIslPCzwNeMxaGwW0BB4wxkQBTwCLrLV1gEWu66Vfv36SvL/6yulIlFKqSPJM4Nbavdba1a7fjwGbgOpAD2Cy626TgZvdFWSxatsWqleH6dOdjkQppYqkQH3gxpgIIA5YDlxqrd3rumkfcGkujxlsjIk3xsSXitVB/PygTx/45hs4fNjpaJRSqtDyncCNMeWB2cAwa+3R7LdZWXgvx8X3rLXvWWubWWubhYeHFynYYtOvH6SmwuzZTkeilFKFlq8EbowJRJL3NGvt567N+40xl7tuvxw44J4Q3SAuDurV024UpZRHy88oFAN8AGyy1r6W7aa5QH/X7/2BL4s/PDcxBvr2hR9/hN27nY5GKaUKJT8t8DbAXcB1xpg1rksX4CWgkzHmD+B613XP0bcvWAvPPCM/lVLKw+S5Kr219ifA5HJzx+INpwTVrQtPPAEvvSS/jxjhdERKKVUgeSZwrzZmDGzfDo8/DldcAbfd5nRESimVb76dwP384KOPpB/87rslibdo4XRUSimVL75VCyUnISEwZw6EhsKzzzodjVJK5ZsmcIDKleHee2Vyz86dTkejlFL5ogk806BB8nPSJGfjUEqpfNIEnikiAjp1kgSenu50NEoplSdN4NkNHiyLHi9c6HQkSimVJ03g2d10E1StCu+/73QkSimVJ03g2QUFwYABUit87948766UUk7SBH6+e++VPvD+/WHuXDhzxumIlFIqR5rAz1enDoweDatWQY8e0qXy+utOR6WUUhfQBJ6TUaNg3z5YsABat4ZHH4UZM5yOSimlzqEJPDeBgXDDDTJLs21b6RtfvtzpqJRS6ixN4HkJDobPP4dq1aRLZdcupyNSSilAE3j+VKkCX38Np05B8+bwwQc62Ucp5ThN4PkVFQX//S9ERspIldhYmDdPF4NQSjlGE3hBNG0KP/8Ms2bByZPQrRs0ayaLI2dkOB2dUsrHaAIvKGOgVy/YtEm6Uo4dk+sxMToFXylVojSBF1ZQEAwcKIn8k0+kf/yGG6BLF9mmlFJupgm8qPz9oU8f2LgRXn0Vli2Dli0hKenC+x49WvLxKaW8libw4hIcLBN+fvpJulXOn7356qsyq/PXX52JTynldTSBF7dGjaRPfMIEOHxYtiUmwlNPSV2VYcN05IpSqlhoAneHUaOkFT5+vFwfPlxGqTzxBCxZIqNWlFKqiDSBu0PjxtCzJ7zxBnzxBXz6KTz5JDz3nNw2YgScPu10lEopD6cJ3F1GjZKTlr17y3JtI0ZAQIC0yhMS4LXXnI5QKeXhNIG7S0wM3HyzTLkfPx7KlJHt110n2194QaodKqVUIWkCd6d334WZM6F793O3jx8PNWvKmPE774SDB52JTynl0TSBu1PVqtKFYsy526+4AtaskZEpM2dCvXrQty+8+SasXetMrEopj6MJ3CnBwfDsszIu/IYbYOlSGDpUimS9/bbT0SmlPIAmcKc1bAjTp8tY8Z07oVMnOeG5bZvTkSmlSjlN4KVJzZpSIMvPDwYN0gqHSqmLyjOBG2MmGWMOGGM2ZNv2jDFmtzFmjevSxb1h+pCaNWWI4eLFchJUKaVykZ8W+EfADTlsf91aG+u6zC/esHzcoEFZXSnbtzsdjVKqlMozgVtrlwCHSyAWlckYmDhRKh326wepqU5HpJQqhYrSB/6gMWadq4slNLc7GWMGG2PijTHxB3W8c/7VqgXvvQf/+5/M6lRKqfMUNoG/DVwJxAJ7gVdzu6O19j1rbTNrbbPw8PBCvpyPuv12uO8+GDtWV/tRSl2gUAncWrvfWpturc0A3geaF29Y6qzx42Wo4V13wd69TkejlCpFCpXAjTGXZ7t6C7Aht/uqIipbVmZrHjsmZWmVUsolP8MIPwF+AeoZYxKNMYOAl40x640x64AOwCNujtO3RUXBI4/IhB9d0Ucp5WJsCa4O06xZMxsfH19ir+dV/voLrrwSrr4avvnG6WiUUiXIGLPKWtvs/O06E9NTVKoE//ynnMxctMjpaJRSpYAmcE/yf/8nwwtHjtRp9kopTeAeJSRElmVbtQqmTnU6GqWUwzSBe5o77oDWrWV8+Ny5TkejlHKQJnBP4+8P8+ZJ3fBevSSJWwvr10t9cV3xXimfEeB0AKoQKlWCb7+Fv/1Nknjt2rBli9zm5wdz5sBNNzkbo1LK7bQF7qkyk3iHDlC9uqzis3UrNGkiU/CXL3c6QqWUm2kL3JNdcsmFNVLmzZM+8m7dYNkyqFPHmdiUUm6nLXBvU7Vq1kSfrl3h6FFn41FKuY0mcG901VXw+eeyrua998pJTqWU19EE7q3atoUXX4RZs+DNN3O+T3IyrFlTsnEppYqNJnBvNnw49OgBjz0Gv/xy7m0JCdC8uZz0/PZbR8JTShWNJnBvZgx89JEslNypEzz5JCQlSau7VSs4cEC6W/r1g127nI5WKVVAmsC9XaVKUvyqe3dZ2ScyUrpXAgPh55/hq6/gzBkZepiS4nS0SqkC0ATuCyIjpZb4+vVw440QFyddKlFRUK8eTJok1x9+GI4fdzpapVQ+aT1wJR55RJZvCwqCdu1kHHm/fqDrmCrlOK0Hri7utdfghx9g6FDYsweGDYMaNeDOO2VCkFKq1NEEroQx0L49vPIKbNwIGzbA4MHSR96mDUyc6HSESqnzaAJXOWvYUMaP794NnTvLYhJLljgdlVIqG03g6uLKl4dPP5WKhz17wvbtTkeklHLRBK7yVqmSdKWkp0uZ2g8/lCJaGzbo0m5KOUirEar8qVMHZs6Em2+GgQOztlevDn36yDhyPz8pabtjB9x6K1x5pXPxKuUDdBihKpjTp2HvXhmp8uefUjRrwQJITT33fs2by9hyP/2Sp1RR5TaMUBO4KrrDh6WEbZky0upesULW7Hz/famGqJQqEk3gquRYK5OBNm2C33+HsDDZfvy4TOEPDnY2PqU8jE7kUSXHGPj3v+HIEfjXv6R75bXXoFo1WQLuzBmnI1TKK2gCV+4REwMPPADvvAONGklJ24YNpV/80Uedjk4pr6AJXLnP6NFw+eUy/PCrryR5Dx8Ob70FU6Y4HZ1SHk+HESr3qVQJNm+GkBDp+wZZJSg+Hv7+d4iOhthYZ2NUyoNpC1y5V4UKWckbICAAZsyQE5sdO8K8ec7FppSHyzOBG2MmGWMOGGM2ZNsWZoz5zhjzh+tnqHvDVF7l0kth8WJZKahbNxgx4sJx5EqpPOU5jNAY0w44Dkyx1jZybXsZOGytfckY8wQQaq0dmdeL6TBCdY7Tp+Xk5ltvQcWK0lovU0a6XiIiZCGKq66SbpZGjaBsWacjVsoRRRoHboyJAL7OlsC3AO2ttXuNMZcDi6219fJ6Hk3gKkdz50ptldOn5ZKUJEWzEhKylnnz85PZnZMnQ926joarVEkr7gT+l7W2kut3AxzJvJ7DYwcDgwFq1arVdMeOHYXdB+VrMjIkia9dC7/+KkMSU1OlOmLnzk5Hp1SJcVsCd10/Yq3Nsx9cW+CqSHbskMWZN2yAZ5+VtT2DgqBcOWmVV6nidIRKuUVuCbywwwj3G2Muz9aFcqBo4SmVD1dcAT//DHfdBaNGXXh7eLhMIHrxRWh2wf+6Ul6nsAl8LtAfeMn188tii0ipiylfXiogbtoEJ07ItPyjR2HLFlkKbsECWQJuwgRZEs4YpyNWym3yTODGmE+A9kAVY0wi8DSSuGcaYwYBO4Db3BmkUucwBqKizt3WpYv8TEqShZjvv1+WgOvcWYprBQdDjx46kkV5Fa1GqLxPRgaMGQNPPy3JO9Ntt8kkIm2VKw+j1QiV7/Dzkz7y/fth2zYZkvjUU7Ki0IwZTkenVLHRFrjyDWlpWTXKN2yQpeCU8hDaAle+LSBAJgGlpMianiXYcFHKXTSBK99Rpw6MGwfffgsPPQSnTjkdkVJFoglc+Zb775fk/Z//QJMmsn6nUh5KE7jyLcbIGPGFC2WNztatoWVLGYZ4550waZL0lyvlATSBK9/UuTOsXw9Dh0olxAMH4McfYdAgaNAApk+XOizz5sHLL8Ps2dpvrkodXZFH+a5KlWSx5UzWytJvo0bBHXdceP++faWgVsWKJRejUhehLXClMhkjxbJ+/RW++EKS9U8/waFD8PzzMo48Lg5WrnQ6UqUAHQeuVP79/DP06wcHD0oN8+uvdzoi5SN0HLhSRdWmjbS+r7oKbrpJToQq5SBN4EoVRNWq8N//Qv360t3yxRcXP7lprZwgVcoNNIErVVBVqsCiRdC4Mdx6q4xaeeEF2LXr3PulpED//rKI8/TpzsSqvJomcKUKIyxMhh1OnCgJ+p//hNq14cEHpcWdnAw33ghTp8rizPfcA0uXOh218jKawJUqrHLlZNz4jz/C1q1w330ycuXKK6FpU6lHPmUKxMdLEr/5Zvj9d6ejVl5ER6EoVZx+/x3+8Q9J3tOnZ41U2bZNZnyWLQtdu8pY8jJlYM8emTB04AAMGQL33qv1ytUFirSocXHRBK58hrUXJuLly6XFvm+fLAOXmir96RERkJ4u48/79IF339XJQuocxb2osVLqYnJqRbdoIbXIM6WlSZlbkFWExo6VWaArV8KXX0LDhiUTq/JY2geulFMCsrWf/PzgySdh8WJZrPm662DzZsdCU55BE7hSpck118APP8jvHTvKyVGlcqEJXKnSpn59GWd+5oy0xKdMkQlD330Hx445HZ0qRbQPXKnSqFEjWTmoUyeZDJSpfn3ZXrOmc7GpUkNb4EqVVk2awM6d8McfMkJl1iwZdnjNNTqeXAHaAleqdCtXTopnAcTGyiShv/0N2raFJ56QVYUOH4bKlaW75eqrITAQkpJg7VqZMRob6+w+KLfRceBKeZotWySJ79gh18uXl5Er1krCr1QJdu+W2/z8ZFz5vfc6F68qMh0HrpS3qFdPulCSkyVZZ7a4f/xRTn4ePQrR0XIZP16m+B8+DI8/7nTkqphpAlfKEwUFQXh41vXKlaUy4q23nnu/Dh3kJOjIkTJdf+xY8Pcv2ViV22gCV8qbBQXBxx9Lgn/1VVi9Wmq0XHaZ3L53r6w0VK4chIZCjRpyUR5BE7hS3s7fH/79b2jWDP7v/+Sk5siRMhzx229lGn8mY6QEbk6LOqtSR4cRKuUrBgyQOithYfDoo7Bxo4xkWbECli2D+fNldMvAgXJdlXraAlfKlzRsCKtWydjyRo1klEp2zZtL2dubb5bqiZGR0nf+889ycjQlRaooVq0qVRQjIqSionJEkRK4MSYBOAakA2k5DXNRSpUyZcrICJWcVK4MX38tSfz666Ws7Zo1F3++nj1lqGLlysUfq7qo4uhC6WCtjdXkrZSXqFcPZs+WuiuVKsGYMfDLL1JYKzFRTnyuWQNz5sjiFXPnygfC99/L8MZnn5X1QkeNcnpPvF6RJvK4WuDNrLWH8nN/ncijlBf69Vfo1y+r/K0xMpJlzx5Ytw6iopyNzwu4ZUUeY8x24AhggXette/lcJ/BwGCAWrVqNd2ROXtMKeU9Tp6USUMhIXD77RAcLCUAWrWCBQucjs7juSuBV7fW7jbGVAW+Ax6y1i7J7f7aAlfKh7z2Gjz2mIxuufHGi983IwPWr4effoKlS+VE6SefyDh2lWsCL1IfuLV2t+vnAeALoHlRnk8p5UUefFBa4Y89Jgk5N7/8IqNfYmPlMT/+CJ9/Dm+/XXKxeqhCJ3BjTDljTIXM34HOwIaLP0op5TOCgmDcONi0CUaPlgUqstuxQ6b5t24tJ0bffRcSEqTvvFMnecyRI46EDsgHyaRJUiSslCpKC/xS4CdjzFpgBTDPWvtN8YSllPIK3bvL5fnnoVYteOop+M9/pKZ5RATMmCGTibZsgcGD4Yor5CToK69I8n7hhaznWr0a/v53GZ/ubkePQu/eMGgQ9Okj1R5LI2ttiV2aNm1qlVI+JiPD2m+/tfamm6w1xlqwtmFDa8eMsTYhIffH3XOPtUFB1m7bZu2kSdYGB8tjwdouXaz95Rdrz5xxT8xPPimv88AD1vr5Wdu4sbVbt7rntfIBiLc55FStB66UKjkJCXDqFDRokPd9d++GOnVk6v/u3bLI88SJ0mp/5RUpkQsygejyy6F2bbl/vXoykzR7tcaC2LlTnqNnTykE9u230gqvWBH+/BMCSn4Cu1tGoRSUJnClVIE884xMDHriCXjuuazkefSonOjcuVP6z3fvlolGW7dKX3uZMlIHffjwnNcPTUmROurGXHjbnXfKRKYtW6TbB2RR6VtvlVmqXbu6bXdzowlcKeV5MjIkSUdE5P/+GzfKEMaPP5Zt9evLxKJq1eDgQfjtN9i2Dfr2lftkT+IrV8qImCefPLf/PTUVqleHdu3gs8+KbffySxO4Usq37NgB77wjM0QTE+VSpUpW982sWfDee9JSBzlp2r69tOj//FO6TLJ77DF4801p7Re2e6aQdEk1pZRvueIKePHFnG/LyJCEPXSoDGOsVUsmG23eLLVdzk/eAPfcIy37adNg2DDZtmoVTJkiLfbMRTJKkLbAlVK+ad8+iImRVnmVKlIyd/Zs6NEj98e0aJX/NRsAAA1gSURBVCEnYdeulROyLVpIt0x4OHzwAdx0k1tCdctMTKWU8liXXSarD/32m0zhnzbt4skbpBW+fj38979yMjMtDb78UvrHu3eH+++XujAlRFvgSinf9vHHUja3W7e87/vXXzJk0VpIT4fvvpN+8zNnpHzuK6/IQhkzZ2b1taekwKJFcO21ULZsoULUFrhSSuXkzjvzl7xBEn3PnpKw33tPkjdI9cWXX4ZvvoH9+2X90ZdfhnvvlZZ+ly5yWzHTFrhSShXE4cNSA71jx5xv37NHFoVevBgqVJBumdtvl/ouwcGFekkdhaKUUsUhLCz35A0y3vz77+VEZ1SU1Eh3E03gSilV3Pz9oUkTt7+M9oErpZSH0gSulFIeShO4UkoVUGlZ40ETuFLK45w4AR99JJMiS9KZMzBggJzHHDYMfv+9ZF//fJrAlVKl1qlTUlvqfEOGyKTIzJIk2SUnX3wJzsJKSpKRgJMny/nJt96SsuFdukjtq5wcOQKffipJPyGh+GPSUShKqVIlJUXmvHz6qdSVOn5cigA++KDcPnmyzICPipK5NH/7m5Tqzrxt0CAZbt2yJbRpI9Vi9+6V+TXVqkn5kubNJcn/9ptc/vhDSolv2wanT8saEVWqwKWXSp2rmjXluXftgunTpRLtvn2yvsS4cVJS5eWX5YNl714pdDh7NixbJnWzwsKgX7/8V8XNL53Io5QqFTLXNX73XUmOYWEy6XH3bpg/X9Zz6NlTJjk2bw4LFkDbtpJ4166VZP/gg9ChAzRuDEuWyHaQWlNVq0pp8aNHz31df38pXHjllXIpU0bm6iQlSUy7dsGBA/IcX3whHwrZJSbKhMuFCyVB79ghfeQxMVLbqksXidffv/B/G4+uB75jBxw6JJ+kxkC5chAZKQtqKKVKl/37YexYScCPPCLvV5CW9KuvyiL1tWplrV+8fr1cli+X2lA33ggPPACdO8t7PDUVBg7MKlkSECCJuVo16bqIi4PQUEm0PXrIimuZc2dOnoSgoKyFfDIyZKGdlSvlPlFRsgpbXhMkT52SBBwUlPPt1kprfPp0+QC5/XbpXikuuSVwj1jUeMiQrLVMMy8BAdY2aGDt7bdb++GH1u7fX6inVsqnnTpl7YoV1p44UbDHbd5s7dNPWxsba2337tZOnmztwYPWvvGGtRUryvsTrK1e3dqpU62dONHayy6TbZGR565PfMkl1l5zjbXDh1v7++85v156urUPPWStv7+18+efe9vkyfI8/fpZm5JSqD9DqYcnL2q8fr2cAMjIkEOenCx11zdtghUr5GuOMXJioUGDrLVNO3WSPiylvM2ZM/JeKOws7ZMnpavi5ZeluyIoSPqMmzeXgnuJidKSDg6Wch5ly8prHj8u5a+3bJH33DXXwPbtcv9MnTtLn/XBg/Dww7LmAci6Ca+9Jn3QGRny/OnpUok1p6Upc3L0aM5rLfz5p7zv/bx0WIZHd6FcjLWwZo2sNfrDD9IftmuXbDcGWrWSr1WtW0NsLJQvX6wvr1Sx+t//5KTYlVdC//7yFR/k/3nvXuln/fJLWSj91Cnp161VC666CqKj5RIZKd0W5crJCcGdO6UbcudOeW/s2iVrFxw8KF/3s5e4XrNGTt7VqCFF9FJSJGmfOCEfFuXLS0Jv3x5uu026MTIypEti/nzpzujRIyshZ2TI2sOBgVIuO7+JWp3LaxN4Ts6ckTPLX30Fc+ZI4TCQf546daQ/7NQpOdscHi59cVdcIScdWrWSVry3fpIr56WnS6G6776TBBgdLf+HL7wgfahhYfItMz1dTtiFhMg6vUeOyONr1JAkedllWcl5yxb5mZcKFWRERYMG0j99/gm5zIaPKl18KoGfb98+iI+Xr3Lr1sk/aJky8vXwwAH5x9++HY4dk/tXrChvqPR0uZQpIydJQkPltnLlpCUSHJx1YtXPTy7GyNfRsDAZilS+vJyYSUmRN0dYmDx3WJjcVrasvEH1TXOu1FRpaSYkQO/e8gFbXJKSJBmmpsqxCQzMarEmJ2e1VP395Wt5ZKQc9xMn5LJjh/w/xcfLaIWoKKnhX7myNBw2bJDWbYsW0lKNiZH/r40bpctv9mzpPvDzkxZqppAQGD4cRo6U15k+XYbSBQXJa0RFSZdFXFzO/y9//SUt6cREaaCcPCn7UKtW1uWSS4rv76hKjk8n8PywVsaC/u9/cjY8OTkrKZ86JW/4I0ekDy7zjZzZD2ltVv98Rsa5b8r88PeX4Ud16shX50sukTdzmTKS6DOX7Mscmxoamvs3hNRU6UbavFniaNBAvl4HBso+7dolH1SZHzD+/lmtuL/+kmR11VVZ67OeOSMXyPp7hIRkDYlKS5MRQgcPyvZKleSSfYTQ8ePyGjt3ygdZ2bJySU2VZHrokPyeuX39ellecN8+ebwxcP310KtX1oImxsg3qcBAiSXzQ/LUKXmdhAQZfla9uvwNIiJg9WrpgsgcWlZU1avL8diyRb7NZapZU/6+69df+L8QEiIrcfXpIz8zk+7WrTLcrDg/qJT30AReglJT5Y15+LAky8BAaUVZK9sOHZLEdfKkXJKTZQLB77/Lz+PHpeWfG2Ok9V6+vLQajZHklZIiiTQt7dz7BwTIh0HmN4z8CAyU58nt3yM4WC7HjuV8Hz8/Sax+flkfAPlljCS3v/9dWp1TpsCHH0pizu/jq1eX7oldu7Jm8gUGSpdB586SZAMCJMbU1KwP5QoVslqraWnScs48JpnfvC67DJo2zfqQS0+X+xw6JB8WlSrJ9qNHZanFjRvlg7lhQ/kZoNPnVAFpAvcwaWmS3I8cyWrhJiVltViPHZOkcvy43D8oSBJU1aqSROrXl0S2aZN8rT9xQpJWrVqSpI4ckQ+T1FTZHhEhLf/t2+WbSGKiPGdmVxNkLQN4+rTEdvq0JKtLL5WW6JkzWd9UUlKk9ZmeLq3RzKRYpkxWsgwKkm8BlSvL75kfaKGhsuxgdunp8i0hs0WbkSF/o8xLYKBcgoMlcWcf15v5AXnVVbLvSnkaTeBKKeWhdFFjpZTyMkVK4MaYG4wxW4wxfxpjniiuoJRSSuWt0AncGOMP/Ae4EYgC+hpjooorMKWUUhdXlBZ4c+BPa+02a20KMAPoUTxhKaWUyktREnh1YFe264mubecwxgw2xsQbY+IPHjxYhJdTSimVndtPYlpr37PWNrPWNgsPD3f3yymllM8oSgLfDdTMdr2Ga5tSSqkSUJQEvhKoY4yJNMYEAX2AucUTllJKqbwUaSKPMaYLMB7wByZZa8fkcf+DQD5qpuWoCnCokI/1ZL643764z+Cb++2L+wwF3+8rrLUX9EGX6EzMojDGxOc0E8nb+eJ+++I+g2/uty/uMxTffutMTKWU8lCawJVSykN5UgJ/z+kAHOKL++2L+wy+ud++uM9QTPvtMX3gSimlzuVJLXCllFLZaAJXSikP5REJ3BfK1hpjahpjfjDG/GaM2WiMedi1PcwY850x5g/Xz1CnYy1uxhh/Y8yvxpivXdcjjTHLXcf7U9dEMa9ijKlkjPnMGLPZGLPJGNPK24+1MeYR1//2BmPMJ8aYEG881saYScaYA8aYDdm25XhsjZjg2v91xpgmBXmtUp/AfahsbRrwmLU2CmgJPODazyeARdbaOsAi13Vv8zCwKdv1scDr1tqrgCPAIEeicq83gG+stfWBGGT/vfZYG2OqA0OBZtbaRsjkvz5457H+CLjhvG25HdsbgTquy2Dg7YK8UKlP4PhI2Vpr7V5r7WrX78eQN3R1ZF8nu+42GbjZmQjdwxhTA+gKTHRdN8B1wGeuu3jjPl8CtAM+ALDWplhr/8LLjzUQAJQxxgQAZYG9eOGxttYuAQ6ftzm3Y9sDmGLF/4BKxpjzVoTNnSck8HyVrfUmxpgIIA5YDlxqrXWtq84+4FKHwnKX8cDjgGu5YioDf1lr01zXvfF4RwIHgQ9dXUcTjTHl8OJjba3dDYwDdiKJOxlYhfcf60y5Hdsi5TdPSOA+xRhTHpgNDLPWHs1+m5Uxn14z7tMY0w04YK1d5XQsJSwAaAK8ba2NA05wXneJFx7rUKS1GQlUA8pxYTeDTyjOY+sJCdxnytYaYwKR5D3NWvu5a/P+zK9Urp8HnIrPDdoA3Y0xCUjX2HVI33Al19ds8M7jnQgkWmuXu65/hiR0bz7W1wPbrbUHrbWpwOfI8ff2Y50pt2NbpPzmCQncJ8rWuvp+PwA2WWtfy3bTXKC/6/f+wJclHZu7WGuftNbWsNZGIMf1v9baO4AfgF6uu3nVPgNYa/cBu4wx9VybOgK/4cXHGuk6aWmMKev6X8/cZ68+1tnkdmznAne7RqO0BJKzdbXkzVpb6i9AF+B3YCvwT6fjcdM+XoN8rVoHrHFduiB9wouAP4DvgTCnY3XT/rcHvnb9XhtYAfwJzAKCnY7PDfsbC8S7jvccINTbjzXwLLAZ2ABMBYK98VgDnyD9/KnIt61BuR1bwCCj7LYC65FROvl+LZ1Kr5RSHsoTulCUUkrlQBO4Ukp5KE3gSinloTSBK6WUh9IErpRSHkoTuFJKeShN4Eop5aH+H4ObXoL1hLORAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KQcisxcEpaYB"
      },
      "source": [
        "def get_tensor_from_text_test(text,dataset):\n",
        "    word_list = []\n",
        "    train_word_to_index_dict = get_word_to_index(dataset['text'])\n",
        "    text = text.lower()\n",
        "    words = word_tokenize(text)\n",
        "    words = [word for word in words if word in train_word_to_index_dict.keys()]\n",
        "    for word in words:\n",
        "        word_list.append(train_word_to_index_dict[word])\n",
        "    if len(word_list) > TEXT_LENGTH:\n",
        "        word_list = word_list[:TEXT_LENGTH]\n",
        "    else:\n",
        "        word_list.extend([0]*(TEXT_LENGTH-len(word_list)))\n",
        "    \n",
        "    tensor_list = torch.tensor(word_list, device=device, dtype=torch.long)\n",
        "    return tensor_list\n",
        "\n",
        "class WeebitTestDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self,test_dataset,train_dataset):\n",
        "        self.dataset = test_dataset\n",
        "        self.train_dataset = train_dataset \n",
        "    \n",
        "    def __getitem__(self,index): \n",
        "        text = self.dataset['text'].iloc[index]\n",
        "        x = get_tensor_from_text_test(text,self.train_dataset)\n",
        "        y = torch.tensor(self.dataset['readability'].iloc[index],dtype=torch.float,device=device)\n",
        "        return x,y\n",
        "    \n",
        "    def __len__(self):\n",
        "        return len(self.dataset)\n",
        "\n",
        "\n",
        "def test(dataset):\n",
        "  testloader = torch.utils.data.DataLoader(\n",
        "                      dataset,\n",
        "                      batch_size=1)\n",
        "  with torch.no_grad():   \n",
        "    model = torch.load('/content/drive/MyDrive/Readability_Research_Paper/models/parallel_lstm_3_cnn/lstm_97.pkl')\n",
        "    model.eval()\n",
        "    test_correct = 0\n",
        "    target_list = []\n",
        "    output_list = []\n",
        "    for i, data in enumerate(testloader, 0):\n",
        "      inputs, targets = data\n",
        "      # print(targets.item())\n",
        "      target_list.append(targets.item())\n",
        "      targets = torch.tensor(targets,dtype=torch.long,device=device)\n",
        "      outputs = model(inputs)\n",
        "      output_ids = torch.argmax(outputs, dim=1)\n",
        "      output_list.append(output_ids.item())\n",
        "      # outputs = outputs.squeeze(1)\n",
        "      test_correct += (output_ids == targets).float().sum()\n",
        "\n",
        "    test_accuracy = 100 * test_correct / len(dataset)\n",
        "    confusion_matrix_calc = confusion_matrix(target_list,output_list)\n",
        "    classification_report_calc = classification_report(target_list,output_list)\n",
        "    return test_accuracy, confusion_matrix_calc, classification_report_calc"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JXngB7mm9G9H",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1c154f9e-7415-44ff-bc36-91c20ed46c01"
      },
      "source": [
        "weebit_test_dataset = WeebitTestDataset(test_dataset, train_dataset)\n",
        "accuracy, confusion_matrix_calc, classification_report_calc = test(weebit_test_dataset)"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:46: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OIP5MhfG9XjO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7e0133ea-67de-44b3-c020-307d6d32d8a4"
      },
      "source": [
        "print(\"Testing accuracy is: \" + str(accuracy.item()))\n",
        "print(\"\\nConfusion Matrix\")\n",
        "print(confusion_matrix_calc)\n",
        "print(\"\\nClassification Report\")\n",
        "print(classification_report_calc)"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Testing accuracy is: 60.60000228881836\n",
            "\n",
            "Confusion Matrix\n",
            "[[66 16 14  3  1]\n",
            " [39 30 31  0  0]\n",
            " [25 23 51  1  0]\n",
            " [ 2  1  1 75 21]\n",
            " [ 0  0  2 17 81]]\n",
            "\n",
            "Classification Report\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.50      0.66      0.57       100\n",
            "         1.0       0.43      0.30      0.35       100\n",
            "         2.0       0.52      0.51      0.51       100\n",
            "         3.0       0.78      0.75      0.77       100\n",
            "         4.0       0.79      0.81      0.80       100\n",
            "\n",
            "    accuracy                           0.61       500\n",
            "   macro avg       0.60      0.61      0.60       500\n",
            "weighted avg       0.60      0.61      0.60       500\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rtl34zWrN6Lz"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}