Copy_of_major_wmt4.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "473dd75b8f9c46d997e7136e614c2d8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9feab48c67d24f948075684a2a26a217",
              "IPY_MODEL_3560154ac4a84b1ab56465c80c753538",
              "IPY_MODEL_3259a3cad58c407daca409ee3d321e6d"
            ],
            "layout": "IPY_MODEL_23a90dcf563a40fa8480166377c510f5"
          }
        },
        "9feab48c67d24f948075684a2a26a217": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fcd19ad63bb34b30b04b09f445222e96",
            "placeholder": "​",
            "style": "IPY_MODEL_0ffcf02892534d6593ae806c62a1de0b",
            "value": "100%"
          }
        },
        "3560154ac4a84b1ab56465c80c753538": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_28461c2fcb8b4359a4201858342da740",
            "max": 33395,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_878748c12a5b429e986ea516932f2b3f",
            "value": 33395
          }
        },
        "3259a3cad58c407daca409ee3d321e6d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6ffcdcaefbd44ba594b84c6e9c178edc",
            "placeholder": "​",
            "style": "IPY_MODEL_6f34900b4c4844eaa2515d7f559b5f1f",
            "value": " 33395/33395 [2:09:05&lt;00:00,  4.57it/s]"
          }
        },
        "23a90dcf563a40fa8480166377c510f5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fcd19ad63bb34b30b04b09f445222e96": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0ffcf02892534d6593ae806c62a1de0b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "28461c2fcb8b4359a4201858342da740": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "878748c12a5b429e986ea516932f2b3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6ffcdcaefbd44ba594b84c6e9c178edc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6f34900b4c4844eaa2515d7f559b5f1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f3m7l1YS7XXF"
      },
      "source": [
        "# Download Transformer\n",
        "Modules"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2a609pb6dKqA",
        "outputId": "997d1385-d3ee-4f22-bb95-d0c4c46f1411"
      },
      "source": [
        "!pip install transformers sentencepiece datasets"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.34.1)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (0.1.99)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (2.14.6)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.4)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.17.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.15,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.14.1)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (9.0.0)\n",
            "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.3.7)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets) (0.70.15)\n",
            "Requirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.8.6)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (3.3.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (4.5.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.7.22)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.3.post1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F5m4W5h7cmDn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dddb41fa-ce76-45b5-f7c4-717aa10ba16f"
      },
      "source": [
        "from datasets import load_dataset\n",
        "from google.colab import drive\n",
        "from IPython.display import display\n",
        "from IPython.html import widgets\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import torch\n",
        "from torch import optim\n",
        "from torch.nn import functional as F\n",
        "from transformers import AdamW, AutoModelForSeq2SeqLM, AutoTokenizer\n",
        "from transformers import get_linear_schedule_with_warmup\n",
        "from tqdm import tqdm_notebook\n",
        "\n",
        "sns.set()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/IPython/html.py:12: ShimWarning: The `IPython.html` package has been deprecated since IPython 4.0. You should import from `notebook` instead. `IPython.html.widgets` has moved to `ipywidgets`.\n",
            "  warn(\"The `IPython.html` package has been deprecated since IPython 4.0. \"\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y8IM3iH4SmzI",
        "outputId": "6485bfc1-4a85-4151-963e-f817e58d1087"
      },
      "source": [
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TmkmzYNOQ9xC"
      },
      "source": [
        "model_repo = 'google/mt5-small'\n",
        "model_path = '/content/gdrive/My Drive/mt5_translation(bn-en).pt'\n",
        "max_seq_len = 20"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fdx0IvJW7dDz"
      },
      "source": [
        "# Load Tokenizer & Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZAXKisA8FuAR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e6d6a7ea-e5c2-4afa-a410-09735273e727"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(model_repo)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thouroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/convert_slow_tokenizer.py:473: UserWarning: The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "En-NjwOrUgJN"
      },
      "source": [
        "model = AutoModelForSeq2SeqLM.from_pretrained(model_repo)\n",
        "model = model.cuda()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hprdz6wlMy8f"
      },
      "source": [
        "# Overview and Quick Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QsO9nJEyMAPq",
        "outputId": "9cef44dc-b657-46bb-96cf-d7268b2d07fd"
      },
      "source": [
        "token_ids = tokenizer.encode(\n",
        "    '<jp> This will be translated to Japanese!',\n",
        "    return_tensors='pt').cuda()\n",
        "print(token_ids)\n",
        "\n",
        "model_out = model.generate(token_ids)\n",
        "print(model_out)\n",
        "\n",
        "output_text = tokenizer.convert_tokens_to_string(\n",
        "    tokenizer.convert_ids_to_tokens(model_out[0]))\n",
        "print(output_text)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[ 1042,  3889,   669,  1494,   898,   390, 37194,   285,   288, 30865,\n",
            "           309,     1]], device='cuda:0')\n",
            "tensor([[     0, 250099,      1]], device='cuda:0')\n",
            "<pad> <extra_id_0></s>\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py:1260: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wHNfsuI57iGg"
      },
      "source": [
        "# Test Tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a8JmwhL1d6g7",
        "outputId": "b2be3f21-163e-4942-8c14-6cecf8892d12"
      },
      "source": [
        "example_input_str = '<jp> This is just a test nbuig.'\n",
        "# example_input_str = 'これは普通のテスト'\n",
        "input_ids = tokenizer.encode(example_input_str, return_tensors='pt')\n",
        "print('Input IDs:', input_ids)\n",
        "\n",
        "tokens = tokenizer.convert_ids_to_tokens(input_ids[0])\n",
        "print('Tokens:', tokens)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Input IDs: tensor([[1042, 3889,  669, 1494,  339, 1627,  259,  262, 2978,  259,  272, 1982,\n",
            "         1315,  260,    1]])\n",
            "Tokens: ['▁<', 'jp', '>', '▁This', '▁is', '▁just', '▁', 'a', '▁test', '▁', 'n', 'bu', 'ig', '.', '</s>']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mwd4V8Krjga5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "40a1a9e3-7a2a-440c-8c5d-b526d9c8d08e"
      },
      "source": [
        "sorted(tokenizer.vocab.items(), key=lambda x: x[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('<pad>', 0),\n",
              " ('</s>', 1),\n",
              " ('<unk>', 2),\n",
              " ('<0x00>', 3),\n",
              " ('<0x01>', 4),\n",
              " ('<0x02>', 5),\n",
              " ('<0x03>', 6),\n",
              " ('<0x04>', 7),\n",
              " ('<0x05>', 8),\n",
              " ('<0x06>', 9),\n",
              " ('<0x07>', 10),\n",
              " ('<0x08>', 11),\n",
              " ('<0x09>', 12),\n",
              " ('<0x0A>', 13),\n",
              " ('<0x0B>', 14),\n",
              " ('<0x0C>', 15),\n",
              " ('<0x0D>', 16),\n",
              " ('<0x0E>', 17),\n",
              " ('<0x0F>', 18),\n",
              " ('<0x10>', 19),\n",
              " ('<0x11>', 20),\n",
              " ('<0x12>', 21),\n",
              " ('<0x13>', 22),\n",
              " ('<0x14>', 23),\n",
              " ('<0x15>', 24),\n",
              " ('<0x16>', 25),\n",
              " ('<0x17>', 26),\n",
              " ('<0x18>', 27),\n",
              " ('<0x19>', 28),\n",
              " ('<0x1A>', 29),\n",
              " ('<0x1B>', 30),\n",
              " ('<0x1C>', 31),\n",
              " ('<0x1D>', 32),\n",
              " ('<0x1E>', 33),\n",
              " ('<0x1F>', 34),\n",
              " ('<0x20>', 35),\n",
              " ('<0x21>', 36),\n",
              " ('<0x22>', 37),\n",
              " ('<0x23>', 38),\n",
              " ('<0x24>', 39),\n",
              " ('<0x25>', 40),\n",
              " ('<0x26>', 41),\n",
              " ('<0x27>', 42),\n",
              " ('<0x28>', 43),\n",
              " ('<0x29>', 44),\n",
              " ('<0x2A>', 45),\n",
              " ('<0x2B>', 46),\n",
              " ('<0x2C>', 47),\n",
              " ('<0x2D>', 48),\n",
              " ('<0x2E>', 49),\n",
              " ('<0x2F>', 50),\n",
              " ('<0x30>', 51),\n",
              " ('<0x31>', 52),\n",
              " ('<0x32>', 53),\n",
              " ('<0x33>', 54),\n",
              " ('<0x34>', 55),\n",
              " ('<0x35>', 56),\n",
              " ('<0x36>', 57),\n",
              " ('<0x37>', 58),\n",
              " ('<0x38>', 59),\n",
              " ('<0x39>', 60),\n",
              " ('<0x3A>', 61),\n",
              " ('<0x3B>', 62),\n",
              " ('<0x3C>', 63),\n",
              " ('<0x3D>', 64),\n",
              " ('<0x3E>', 65),\n",
              " ('<0x3F>', 66),\n",
              " ('<0x40>', 67),\n",
              " ('<0x41>', 68),\n",
              " ('<0x42>', 69),\n",
              " ('<0x43>', 70),\n",
              " ('<0x44>', 71),\n",
              " ('<0x45>', 72),\n",
              " ('<0x46>', 73),\n",
              " ('<0x47>', 74),\n",
              " ('<0x48>', 75),\n",
              " ('<0x49>', 76),\n",
              " ('<0x4A>', 77),\n",
              " ('<0x4B>', 78),\n",
              " ('<0x4C>', 79),\n",
              " ('<0x4D>', 80),\n",
              " ('<0x4E>', 81),\n",
              " ('<0x4F>', 82),\n",
              " ('<0x50>', 83),\n",
              " ('<0x51>', 84),\n",
              " ('<0x52>', 85),\n",
              " ('<0x53>', 86),\n",
              " ('<0x54>', 87),\n",
              " ('<0x55>', 88),\n",
              " ('<0x56>', 89),\n",
              " ('<0x57>', 90),\n",
              " ('<0x58>', 91),\n",
              " ('<0x59>', 92),\n",
              " ('<0x5A>', 93),\n",
              " ('<0x5B>', 94),\n",
              " ('<0x5C>', 95),\n",
              " ('<0x5D>', 96),\n",
              " ('<0x5E>', 97),\n",
              " ('<0x5F>', 98),\n",
              " ('<0x60>', 99),\n",
              " ('<0x61>', 100),\n",
              " ('<0x62>', 101),\n",
              " ('<0x63>', 102),\n",
              " ('<0x64>', 103),\n",
              " ('<0x65>', 104),\n",
              " ('<0x66>', 105),\n",
              " ('<0x67>', 106),\n",
              " ('<0x68>', 107),\n",
              " ('<0x69>', 108),\n",
              " ('<0x6A>', 109),\n",
              " ('<0x6B>', 110),\n",
              " ('<0x6C>', 111),\n",
              " ('<0x6D>', 112),\n",
              " ('<0x6E>', 113),\n",
              " ('<0x6F>', 114),\n",
              " ('<0x70>', 115),\n",
              " ('<0x71>', 116),\n",
              " ('<0x72>', 117),\n",
              " ('<0x73>', 118),\n",
              " ('<0x74>', 119),\n",
              " ('<0x75>', 120),\n",
              " ('<0x76>', 121),\n",
              " ('<0x77>', 122),\n",
              " ('<0x78>', 123),\n",
              " ('<0x79>', 124),\n",
              " ('<0x7A>', 125),\n",
              " ('<0x7B>', 126),\n",
              " ('<0x7C>', 127),\n",
              " ('<0x7D>', 128),\n",
              " ('<0x7E>', 129),\n",
              " ('<0x7F>', 130),\n",
              " ('<0x80>', 131),\n",
              " ('<0x81>', 132),\n",
              " ('<0x82>', 133),\n",
              " ('<0x83>', 134),\n",
              " ('<0x84>', 135),\n",
              " ('<0x85>', 136),\n",
              " ('<0x86>', 137),\n",
              " ('<0x87>', 138),\n",
              " ('<0x88>', 139),\n",
              " ('<0x89>', 140),\n",
              " ('<0x8A>', 141),\n",
              " ('<0x8B>', 142),\n",
              " ('<0x8C>', 143),\n",
              " ('<0x8D>', 144),\n",
              " ('<0x8E>', 145),\n",
              " ('<0x8F>', 146),\n",
              " ('<0x90>', 147),\n",
              " ('<0x91>', 148),\n",
              " ('<0x92>', 149),\n",
              " ('<0x93>', 150),\n",
              " ('<0x94>', 151),\n",
              " ('<0x95>', 152),\n",
              " ('<0x96>', 153),\n",
              " ('<0x97>', 154),\n",
              " ('<0x98>', 155),\n",
              " ('<0x99>', 156),\n",
              " ('<0x9A>', 157),\n",
              " ('<0x9B>', 158),\n",
              " ('<0x9C>', 159),\n",
              " ('<0x9D>', 160),\n",
              " ('<0x9E>', 161),\n",
              " ('<0x9F>', 162),\n",
              " ('<0xA0>', 163),\n",
              " ('<0xA1>', 164),\n",
              " ('<0xA2>', 165),\n",
              " ('<0xA3>', 166),\n",
              " ('<0xA4>', 167),\n",
              " ('<0xA5>', 168),\n",
              " ('<0xA6>', 169),\n",
              " ('<0xA7>', 170),\n",
              " ('<0xA8>', 171),\n",
              " ('<0xA9>', 172),\n",
              " ('<0xAA>', 173),\n",
              " ('<0xAB>', 174),\n",
              " ('<0xAC>', 175),\n",
              " ('<0xAD>', 176),\n",
              " ('<0xAE>', 177),\n",
              " ('<0xAF>', 178),\n",
              " ('<0xB0>', 179),\n",
              " ('<0xB1>', 180),\n",
              " ('<0xB2>', 181),\n",
              " ('<0xB3>', 182),\n",
              " ('<0xB4>', 183),\n",
              " ('<0xB5>', 184),\n",
              " ('<0xB6>', 185),\n",
              " ('<0xB7>', 186),\n",
              " ('<0xB8>', 187),\n",
              " ('<0xB9>', 188),\n",
              " ('<0xBA>', 189),\n",
              " ('<0xBB>', 190),\n",
              " ('<0xBC>', 191),\n",
              " ('<0xBD>', 192),\n",
              " ('<0xBE>', 193),\n",
              " ('<0xBF>', 194),\n",
              " ('<0xC0>', 195),\n",
              " ('<0xC1>', 196),\n",
              " ('<0xC2>', 197),\n",
              " ('<0xC3>', 198),\n",
              " ('<0xC4>', 199),\n",
              " ('<0xC5>', 200),\n",
              " ('<0xC6>', 201),\n",
              " ('<0xC7>', 202),\n",
              " ('<0xC8>', 203),\n",
              " ('<0xC9>', 204),\n",
              " ('<0xCA>', 205),\n",
              " ('<0xCB>', 206),\n",
              " ('<0xCC>', 207),\n",
              " ('<0xCD>', 208),\n",
              " ('<0xCE>', 209),\n",
              " ('<0xCF>', 210),\n",
              " ('<0xD0>', 211),\n",
              " ('<0xD1>', 212),\n",
              " ('<0xD2>', 213),\n",
              " ('<0xD3>', 214),\n",
              " ('<0xD4>', 215),\n",
              " ('<0xD5>', 216),\n",
              " ('<0xD6>', 217),\n",
              " ('<0xD7>', 218),\n",
              " ('<0xD8>', 219),\n",
              " ('<0xD9>', 220),\n",
              " ('<0xDA>', 221),\n",
              " ('<0xDB>', 222),\n",
              " ('<0xDC>', 223),\n",
              " ('<0xDD>', 224),\n",
              " ('<0xDE>', 225),\n",
              " ('<0xDF>', 226),\n",
              " ('<0xE0>', 227),\n",
              " ('<0xE1>', 228),\n",
              " ('<0xE2>', 229),\n",
              " ('<0xE3>', 230),\n",
              " ('<0xE4>', 231),\n",
              " ('<0xE5>', 232),\n",
              " ('<0xE6>', 233),\n",
              " ('<0xE7>', 234),\n",
              " ('<0xE8>', 235),\n",
              " ('<0xE9>', 236),\n",
              " ('<0xEA>', 237),\n",
              " ('<0xEB>', 238),\n",
              " ('<0xEC>', 239),\n",
              " ('<0xED>', 240),\n",
              " ('<0xEE>', 241),\n",
              " ('<0xEF>', 242),\n",
              " ('<0xF0>', 243),\n",
              " ('<0xF1>', 244),\n",
              " ('<0xF2>', 245),\n",
              " ('<0xF3>', 246),\n",
              " ('<0xF4>', 247),\n",
              " ('<0xF5>', 248),\n",
              " ('<0xF6>', 249),\n",
              " ('<0xF7>', 250),\n",
              " ('<0xF8>', 251),\n",
              " ('<0xF9>', 252),\n",
              " ('<0xFA>', 253),\n",
              " ('<0xFB>', 254),\n",
              " ('<0xFC>', 255),\n",
              " ('<0xFD>', 256),\n",
              " ('<0xFE>', 257),\n",
              " ('<0xFF>', 258),\n",
              " ('▁', 259),\n",
              " ('.', 260),\n",
              " (',', 261),\n",
              " ('a', 262),\n",
              " ('s', 263),\n",
              " ('-', 264),\n",
              " ('e', 265),\n",
              " ('i', 266),\n",
              " (':', 267),\n",
              " ('o', 268),\n",
              " ('▁de', 269),\n",
              " ('t', 270),\n",
              " (')', 271),\n",
              " ('n', 272),\n",
              " ('u', 273),\n",
              " ('▁(', 274),\n",
              " ('/', 275),\n",
              " ('y', 276),\n",
              " (\"'\", 277),\n",
              " ('en', 278),\n",
              " ('и', 279),\n",
              " ('l', 280),\n",
              " ('▁in', 281),\n",
              " ('m', 282),\n",
              " ('▁la', 283),\n",
              " ('com', 284),\n",
              " ('d', 285),\n",
              " ('r', 286),\n",
              " ('▁the', 287),\n",
              " ('▁to', 288),\n",
              " ('▁en', 289),\n",
              " ('_', 290),\n",
              " ('?', 291),\n",
              " ('、', 292),\n",
              " ('’', 293),\n",
              " ('▁na', 294),\n",
              " ('er', 295),\n",
              " (';', 296),\n",
              " ('c', 297),\n",
              " ('▁A', 298),\n",
              " ('es', 299),\n",
              " ('▁v', 300),\n",
              " ('▁di', 301),\n",
              " ('...', 302),\n",
              " ('▁se', 303),\n",
              " ('▁of', 304),\n",
              " ('▁and', 305),\n",
              " ('。', 306),\n",
              " ('▁|', 307),\n",
              " ('а', 308),\n",
              " ('!', 309),\n",
              " ('▁на', 310),\n",
              " ('\"', 311),\n",
              " ('(', 312),\n",
              " ('▁\"', 313),\n",
              " ('k', 314),\n",
              " ('▁в', 315),\n",
              " ('b', 316),\n",
              " ('▁c', 317),\n",
              " ('g', 318),\n",
              " ('▁que', 319),\n",
              " ('▁S', 320),\n",
              " ('an', 321),\n",
              " ('▁–', 322),\n",
              " ('▁www', 323),\n",
              " ('е', 324),\n",
              " ('p', 325),\n",
              " ('▁m', 326),\n",
              " ('▁sa', 327),\n",
              " ('3', 328),\n",
              " ('x', 329),\n",
              " ('▁b', 330),\n",
              " ('▁d', 331),\n",
              " ('▁for', 332),\n",
              " ('▁1', 333),\n",
              " ('h', 334),\n",
              " ('▁un', 335),\n",
              " ('▁I', 336),\n",
              " ('os', 337),\n",
              " ('2', 338),\n",
              " ('▁is', 339),\n",
              " ('▁le', 340),\n",
              " ('▁و', 341),\n",
              " ('▁do', 342),\n",
              " ('،', 343),\n",
              " ('▁at', 344),\n",
              " ('ed', 345),\n",
              " ('te', 346),\n",
              " ('ing', 347),\n",
              " ('in', 348),\n",
              " ('=', 349),\n",
              " ('▁da', 350),\n",
              " ('▁on', 351),\n",
              " ('▁M', 352),\n",
              " ('1', 353),\n",
              " ('у', 354),\n",
              " ('▁đ', 355),\n",
              " ('▁2', 356),\n",
              " ('A', 357),\n",
              " ('as', 358),\n",
              " ('▁“', 359),\n",
              " ('z', 360),\n",
              " ('é', 361),\n",
              " ('▁el', 362),\n",
              " ('▁P', 363),\n",
              " ('▁B', 364),\n",
              " ('”', 365),\n",
              " ('▁T', 366),\n",
              " ('f', 367),\n",
              " ('de', 368),\n",
              " ('à', 369),\n",
              " ('ng', 370),\n",
              " ('▁C', 371),\n",
              " ('ar', 372),\n",
              " ('▁og', 373),\n",
              " ('▁за', 374),\n",
              " ('▁no', 375),\n",
              " ('ه', 376),\n",
              " ('na', 377),\n",
              " ('।', 378),\n",
              " ('v', 379),\n",
              " ('re', 380),\n",
              " ('▁3', 381),\n",
              " ('▁h', 382),\n",
              " ('▁et', 383),\n",
              " ('▁je', 384),\n",
              " ('j', 385),\n",
              " ('▁il', 386),\n",
              " ('▁#', 387),\n",
              " ('▁с', 388),\n",
              " ('і', 389),\n",
              " ('▁be', 390),\n",
              " ('://', 391),\n",
              " ('▁2018', 392),\n",
              " ('▁per', 393),\n",
              " ('▁th', 394),\n",
              " ('▁si', 395),\n",
              " ('я', 396),\n",
              " ('▁z', 397),\n",
              " ('▁die', 398),\n",
              " ('S', 399),\n",
              " ('▁te', 400),\n",
              " ('▁не', 401),\n",
              " ('▁ال', 402),\n",
              " ('D', 403),\n",
              " ('▁«', 404),\n",
              " ('ne', 405),\n",
              " ('ی', 406),\n",
              " ('da', 407),\n",
              " ('▁k', 408),\n",
              " ('|', 409),\n",
              " ('4', 410),\n",
              " ('о', 411),\n",
              " ('▁K', 412),\n",
              " ('▁du', 413),\n",
              " ('▁w', 414),\n",
              " ('▁E', 415),\n",
              " ('▁me', 416),\n",
              " ('is', 417),\n",
              " ('▁are', 418),\n",
              " ('▁4', 419),\n",
              " ('í', 420),\n",
              " ('▁p', 421),\n",
              " ('ta', 422),\n",
              " ('の', 423),\n",
              " ('C', 424),\n",
              " ('▁по', 425),\n",
              " ('▁del', 426),\n",
              " ('▁ka', 427),\n",
              " ('5', 428),\n",
              " ('et', 429),\n",
              " ('▁5', 430),\n",
              " ('▁D', 431),\n",
              " ('▁ja', 432),\n",
              " ('ы', 433),\n",
              " ('▁V', 434),\n",
              " ('▁para', 435),\n",
              " ('»', 436),\n",
              " ('\",\"', 437),\n",
              " ('us', 438),\n",
              " (']', 439),\n",
              " ('▁al', 440),\n",
              " ('▁N', 441),\n",
              " ('▁der', 442),\n",
              " ('▁O', 443),\n",
              " ('on', 444),\n",
              " ('ة', 445),\n",
              " ('▁да', 446),\n",
              " ('▁H', 447),\n",
              " ('▁ne', 448),\n",
              " ('8', 449),\n",
              " ('▁con', 450),\n",
              " ('6', 451),\n",
              " ('B', 452),\n",
              " ('▁er', 453),\n",
              " ('ul', 454),\n",
              " ('▁by', 455),\n",
              " ('▁у', 456),\n",
              " ('▁yang', 457),\n",
              " ('▁L', 458),\n",
              " ('▁De', 459),\n",
              " ('0', 460),\n",
              " ('▁an', 461),\n",
              " ('ja', 462),\n",
              " ('\\xad', 463),\n",
              " ('▁van', 464),\n",
              " ('▁ה', 465),\n",
              " ('▁za', 466),\n",
              " ('】【', 467),\n",
              " ('le', 468),\n",
              " ('▁dan', 469),\n",
              " ('em', 470),\n",
              " ('á', 471),\n",
              " ('▁und', 472),\n",
              " ('al', 473),\n",
              " ('è', 474),\n",
              " ('▁10', 475),\n",
              " ('to', 476),\n",
              " ('ي', 477),\n",
              " ('E', 478),\n",
              " ('ka', 479),\n",
              " ('▁...', 480),\n",
              " ('w', 481),\n",
              " ('▁på', 482),\n",
              " (').', 483),\n",
              " ('ly', 484),\n",
              " ('▁po', 485),\n",
              " ('▁The', 486),\n",
              " ('7', 487),\n",
              " ('\":\"', 488),\n",
              " ('▁G', 489),\n",
              " ('T', 490),\n",
              " ('▁[', 491),\n",
              " ('la', 492),\n",
              " ('的', 493),\n",
              " ('li', 494),\n",
              " ('9', 495),\n",
              " ('▁ma', 496),\n",
              " ('▁0', 497),\n",
              " ('▁des', 498),\n",
              " ('▁med', 499),\n",
              " ('▁til', 500),\n",
              " ('▁La', 501),\n",
              " ('kan', 502),\n",
              " ('it', 503),\n",
              " ('▁ki', 504),\n",
              " ('no', 505),\n",
              " ('),', 506),\n",
              " ('м', 507),\n",
              " ('َ', 508),\n",
              " ('▁در', 509),\n",
              " ('▁so', 510),\n",
              " ('M', 511),\n",
              " ('▁som', 512),\n",
              " ('▁ke', 513),\n",
              " ('▁with', 514),\n",
              " ('▁F', 515),\n",
              " ('ni', 516),\n",
              " ('▁su', 517),\n",
              " ('▁και', 518),\n",
              " ('▁por', 519),\n",
              " ('▁les', 520),\n",
              " ('▁you', 521),\n",
              " ('si', 522),\n",
              " ('at', 523),\n",
              " ('ti', 524),\n",
              " ('id', 525),\n",
              " ('▁av', 526),\n",
              " ('▁as', 527),\n",
              " ('▁ya', 528),\n",
              " ('▁ve', 529),\n",
              " ('▁den', 530),\n",
              " ('▁R', 531),\n",
              " ('▁ב', 532),\n",
              " ('▁that', 533),\n",
              " ('▁tr', 534),\n",
              " ('は', 535),\n",
              " ('が', 536),\n",
              " ('do', 537),\n",
              " ('N', 538),\n",
              " ('ia', 539),\n",
              " ('\\\\', 540),\n",
              " ('ce', 541),\n",
              " ('▁om', 542),\n",
              " ('й', 543),\n",
              " ('▁се', 544),\n",
              " ('F', 545),\n",
              " ('&', 546),\n",
              " ('L', 547),\n",
              " ('▁م', 548),\n",
              " ('▁&', 549),\n",
              " ('▁د', 550),\n",
              " ('▁det', 551),\n",
              " ('▁от', 552),\n",
              " ('ó', 553),\n",
              " ('▁به', 554),\n",
              " ('▁pa', 555),\n",
              " ('▁من', 556),\n",
              " ('K', 557),\n",
              " ('на', 558),\n",
              " ('P', 559),\n",
              " ('▁ha', 560),\n",
              " ('V', 561),\n",
              " ('▁ch', 562),\n",
              " ('▁In', 563),\n",
              " ('▁W', 564),\n",
              " ('▁„', 565),\n",
              " ('I', 566),\n",
              " ('▁var', 567),\n",
              " ('▁ni', 568),\n",
              " ('se', 569),\n",
              " ('▁6', 570),\n",
              " ('ra', 571),\n",
              " ('ل', 572),\n",
              " ('▁una', 573),\n",
              " ('を', 574),\n",
              " ('▁في', 575),\n",
              " ('▁ta', 576),\n",
              " ('▁http', 577),\n",
              " ('COM', 578),\n",
              " ('am', 579),\n",
              " ('ה', 580),\n",
              " ('▁U', 581),\n",
              " ('R', 582),\n",
              " ('▁з', 583),\n",
              " ('▁re', 584),\n",
              " ('▁op', 585),\n",
              " ('ن', 586),\n",
              " ('т', 587),\n",
              " ('▁har', 588),\n",
              " ('ο', 589),\n",
              " ('H', 590),\n",
              " ('“', 591),\n",
              " ('ek', 592),\n",
              " ('▁ag', 593),\n",
              " ('▁ng', 594),\n",
              " ('▁los', 595),\n",
              " ('{', 596),\n",
              " ('▁och', 597),\n",
              " ('▁2017', 598),\n",
              " ('▁WWW', 599),\n",
              " ('に', 600),\n",
              " ('▁ku', 601),\n",
              " ('ir', 602),\n",
              " ('▁pe', 603),\n",
              " ('un', 604),\n",
              " ('х', 605),\n",
              " ('um', 606),\n",
              " ('▁2019', 607),\n",
              " ('je', 608),\n",
              " ('▁it', 609),\n",
              " ('▁до', 610),\n",
              " ('을', 611),\n",
              " ('ʻ', 612),\n",
              " ('www', 613),\n",
              " ('▁ب', 614),\n",
              " ('▁li', 615),\n",
              " ('но', 616),\n",
              " ('▁7', 617),\n",
              " ('▁»', 618),\n",
              " ('▁ir', 619),\n",
              " ('▁kan', 620),\n",
              " ('G', 621),\n",
              " ('▁het', 622),\n",
              " ('▁ho', 623),\n",
              " ('▁par', 624),\n",
              " ('▁vi', 625),\n",
              " ('・', 626),\n",
              " ('で', 627),\n",
              " ('▁20', 628),\n",
              " ('▁të', 629),\n",
              " ('▁8', 630),\n",
              " ('▁or', 631),\n",
              " ('ا', 632),\n",
              " ('م', 633),\n",
              " ('ie', 634),\n",
              " ('▁В', 635),\n",
              " ('ت', 636),\n",
              " ('ом', 637),\n",
              " ('W', 638),\n",
              " ('▁was', 639),\n",
              " ('την', 640),\n",
              " ('▁के', 641),\n",
              " ('▁En', 642),\n",
              " ('▁af', 643),\n",
              " ('▁12', 644),\n",
              " ('me', 645),\n",
              " ('O', 646),\n",
              " ('nya', 647),\n",
              " ('ma', 648),\n",
              " ('의', 649),\n",
              " ('ki', 650),\n",
              " ('▁cu', 651),\n",
              " ('μ', 652),\n",
              " ('▁No', 653),\n",
              " ('▁2016', 654),\n",
              " ('▁es', 655),\n",
              " ('▁een', 656),\n",
              " ('ки', 657),\n",
              " ('▁mi', 658),\n",
              " ('Ð', 659),\n",
              " ('10', 660),\n",
              " ('▁—', 661),\n",
              " ('ku', 662),\n",
              " ('\":', 663),\n",
              " ('▁J', 664),\n",
              " ('px', 665),\n",
              " ('일', 666),\n",
              " ('▁ל', 667),\n",
              " ('ни', 668),\n",
              " ('>', 669),\n",
              " ('▁15', 670),\n",
              " ('▁‘', 671),\n",
              " ('▁ver', 672),\n",
              " ('▁um', 673),\n",
              " ('▁man', 674),\n",
              " ('▁ko', 675),\n",
              " ('+', 676),\n",
              " ('▁nh', 677),\n",
              " ('η', 678),\n",
              " ('ка', 679),\n",
              " ('ny', 680),\n",
              " ('α', 681),\n",
              " ('▁od', 682),\n",
              " ('▁wa', 683),\n",
              " ('▁ge', 684),\n",
              " ('ов', 685),\n",
              " ('н', 686),\n",
              " ('ten', 687),\n",
              " ('▁С', 688),\n",
              " ('▁מ', 689),\n",
              " ('▁ph', 690),\n",
              " ('▁>', 691),\n",
              " ('▁men', 692),\n",
              " ('▁ber', 693),\n",
              " ('▁του', 694),\n",
              " ('▁از', 695),\n",
              " ('il', 696),\n",
              " ('ch', 697),\n",
              " ('▁bir', 698),\n",
              " ('▁το', 699),\n",
              " ('▁να', 700),\n",
              " ('el', 701),\n",
              " ('▁from', 702),\n",
              " ('▁nu', 703),\n",
              " ('ko', 704),\n",
              " ('st', 705),\n",
              " ('ë', 706),\n",
              " ('▁lo', 707),\n",
              " ('ủ', 708),\n",
              " ('▁az', 709),\n",
              " ('▁dem', 710),\n",
              " ('mi', 711),\n",
              " ('▁va', 712),\n",
              " ('▁att', 713),\n",
              " ('▁this', 714),\n",
              " ('ur', 715),\n",
              " ('▁nie', 716),\n",
              " ('#', 717),\n",
              " ('▁gi', 718),\n",
              " ('▁tu', 719),\n",
              " ('di', 720),\n",
              " ('å', 721),\n",
              " ('ات', 722),\n",
              " ('or', 723),\n",
              " ('▁em', 724),\n",
              " ('と', 725),\n",
              " ('ת', 726),\n",
              " ('▁Na', 727),\n",
              " ('▁am', 728),\n",
              " ('▁из', 729),\n",
              " ('▁11', 730),\n",
              " ('▁pro', 731),\n",
              " ('▁în', 732),\n",
              " ('▁30', 733),\n",
              " ('▁che', 734),\n",
              " ('для', 735),\n",
              " ('▁Z', 736),\n",
              " ('ru', 737),\n",
              " ('▁can', 738),\n",
              " ('ya', 739),\n",
              " ('▁ang', 740),\n",
              " ('ai', 741),\n",
              " ('▁f', 742),\n",
              " ('ga', 743),\n",
              " ('▁+', 744),\n",
              " ('za', 745),\n",
              " ('▁Se', 746),\n",
              " ('이', 747),\n",
              " ('ю', 748),\n",
              " ('▁mit', 749),\n",
              " ('ca', 750),\n",
              " ('▁all', 751),\n",
              " ('▁של', 752),\n",
              " ('ke', 753),\n",
              " ('\",', 754),\n",
              " ('°', 755),\n",
              " ('▁tak', 756),\n",
              " ('ने', 757),\n",
              " ('▁bu', 758),\n",
              " ('▁bo', 759),\n",
              " ('▁zu', 760),\n",
              " ('ą', 761),\n",
              " ('ή', 762),\n",
              " ('▁pour', 763),\n",
              " ('▁Le', 764),\n",
              " ('[', 765),\n",
              " ('▁ت', 766),\n",
              " ('▁ter', 767),\n",
              " ('▁با', 768),\n",
              " ('ci', 769),\n",
              " ('▁és', 770),\n",
              " ('co', 771),\n",
              " ('▁your', 772),\n",
              " ('om', 773),\n",
              " ('▁9', 774),\n",
              " ('▁کے', 775),\n",
              " ('▁not', 776),\n",
              " ('их', 777),\n",
              " ('▁к', 778),\n",
              " ('▁din', 779),\n",
              " ('im', 780),\n",
              " ('q', 781),\n",
              " ('ă', 782),\n",
              " ('▁have', 783),\n",
              " ('▁mai', 784),\n",
              " ('▁{', 785),\n",
              " ('▁pre', 786),\n",
              " ('▁we', 787),\n",
              " ('▁Re', 788),\n",
              " ('▁El', 789),\n",
              " ('▁he', 790),\n",
              " ('ς', 791),\n",
              " ('▁•', 792),\n",
              " ('và', 793),\n",
              " ('Y', 794),\n",
              " ('▁von', 795),\n",
              " ('▁là', 796),\n",
              " ('ې', 797),\n",
              " ('▁ar', 798),\n",
              " ('▁16', 799),\n",
              " ('▁las', 800),\n",
              " ('ú', 801),\n",
              " ('app', 802),\n",
              " ('▁کی', 803),\n",
              " ('▁au', 804),\n",
              " ('▁при', 805),\n",
              " ('U', 806),\n",
              " ('th', 807),\n",
              " ('▁}', 808),\n",
              " ('▁2014', 809),\n",
              " ('▁ba', 810),\n",
              " ('be', 811),\n",
              " ('▁18', 812),\n",
              " ('X', 813),\n",
              " ('▁2015', 814),\n",
              " ('▁2013', 815),\n",
              " ('▁(1)', 816),\n",
              " ('ой', 817),\n",
              " ('▁14', 818),\n",
              " ('▁qu', 819),\n",
              " ('ِ', 820),\n",
              " ('ha', 821),\n",
              " ('▁می', 822),\n",
              " ('man', 823),\n",
              " ('▁met', 824),\n",
              " ('are', 825),\n",
              " ('▁nga', 826),\n",
              " ('▁das', 827),\n",
              " ('▁της', 828),\n",
              " ('‘', 829),\n",
              " ('▁है', 830),\n",
              " ('ية', 831),\n",
              " ('то', 832),\n",
              " ('ь', 833),\n",
              " ('va', 834),\n",
              " ('ba', 835),\n",
              " ('】', 836),\n",
              " ('▁bi', 837),\n",
              " ('日', 838),\n",
              " ('한', 839),\n",
              " ('▁24', 840),\n",
              " ('ر', 841),\n",
              " ('ى', 842),\n",
              " ('▁est', 843),\n",
              " ('▁में', 844),\n",
              " ('lar', 845),\n",
              " ('▁2012', 846),\n",
              " ('▁dengan', 847),\n",
              " ('年', 848),\n",
              " ('▁13', 849),\n",
              " ('▁με', 850),\n",
              " ('▁untuk', 851),\n",
              " ('▁Y', 852),\n",
              " (');', 853),\n",
              " ('▁ini', 854),\n",
              " ('▁ש', 855),\n",
              " ('▁ist', 856),\n",
              " ('ve', 857),\n",
              " ('▁ا', 858),\n",
              " ('▁im', 859),\n",
              " ('this', 860),\n",
              " ('est', 861),\n",
              " ('▁online', 862),\n",
              " ('न', 863),\n",
              " ('▁А', 864),\n",
              " ('▁sur', 865),\n",
              " ('J', 866),\n",
              " ('▁У', 867),\n",
              " ('ך', 868),\n",
              " ('은', 869),\n",
              " ('ado', 870),\n",
              " ('▁ti', 871),\n",
              " ('ہ', 872),\n",
              " ('에', 873),\n",
              " ('ri', 874),\n",
              " ('▁för', 875),\n",
              " ('tu', 876),\n",
              " ('▁25', 877),\n",
              " ('lo', 878),\n",
              " ('」', 879),\n",
              " ('den', 880),\n",
              " ('%', 881),\n",
              " ('▁א', 882),\n",
              " ('د', 883),\n",
              " ('▁את', 884),\n",
              " ('▁có', 885),\n",
              " ('▁pas', 886),\n",
              " ('=\"', 887),\n",
              " ('▁ein', 888),\n",
              " ('ou', 889),\n",
              " ('▁mu', 890),\n",
              " ('月', 891),\n",
              " ('▁что', 892),\n",
              " ('ого', 893),\n",
              " ('*', 894),\n",
              " ('ի', 895),\n",
              " ('ים', 896),\n",
              " ('р', 897),\n",
              " ('▁will', 898),\n",
              " ('▁fa', 899),\n",
              " ('net', 900),\n",
              " ('▁για', 901),\n",
              " ('д', 902),\n",
              " ('ê', 903),\n",
              " ('▁*', 904),\n",
              " ('ُ', 905),\n",
              " ('ada', 906),\n",
              " ('▁qui', 907),\n",
              " ('ới', 908),\n",
              " ('г', 909),\n",
              " ('▁over', 910),\n",
              " ('▁17', 911),\n",
              " ('▁από', 912),\n",
              " ('ها', 913),\n",
              " (',\"', 914),\n",
              " ('ā', 915),\n",
              " ('▁را', 916),\n",
              " ('▁со', 917),\n",
              " ('та', 918),\n",
              " ('▁ser', 919),\n",
              " ('л', 920),\n",
              " ('que', 921),\n",
              " ('▁так', 922),\n",
              " ('▁про', 923),\n",
              " ('ể', 924),\n",
              " ('ok', 925),\n",
              " ('▁To', 926),\n",
              " ('▁σ', 927),\n",
              " ('▁და', 928),\n",
              " ('가', 929),\n",
              " ('ό', 930),\n",
              " ('ción', 931),\n",
              " ('ak', 932),\n",
              " ('ị', 933),\n",
              " ('▁که', 934),\n",
              " ('▁non', 935),\n",
              " ('ן', 936),\n",
              " ('▁је', 937),\n",
              " ('ro', 938),\n",
              " ('「', 939),\n",
              " ('ag', 940),\n",
              " ('ان', 941),\n",
              " ('على', 942),\n",
              " ('▁आ', 943),\n",
              " ('ите', 944),\n",
              " ('да', 945),\n",
              " ('с', 946),\n",
              " ('▁się', 947),\n",
              " ('▁€', 948),\n",
              " ('▁mo', 949),\n",
              " ('▁است', 950),\n",
              " ('▁·', 951),\n",
              " ('ý', 952),\n",
              " ('▁این', 953),\n",
              " ('Р', 954),\n",
              " ('▁if', 955),\n",
              " ('▁für', 956),\n",
              " ('не', 957),\n",
              " ('▁como', 958),\n",
              " ('▁X', 959),\n",
              " ('▁ca', 960),\n",
              " ('▁är', 961),\n",
              " ('ní', 962),\n",
              " ('▁19', 963),\n",
              " ('▁co', 964),\n",
              " ('▁כ', 965),\n",
              " ('▁100', 966),\n",
              " ('ere', 967),\n",
              " ('▁að', 968),\n",
              " ('wa', 969),\n",
              " ('▁cho', 970),\n",
              " ('▁voor', 971),\n",
              " ('▁2020', 972),\n",
              " ('▁میں', 973),\n",
              " ('و', 974),\n",
              " ('▁की', 975),\n",
              " ('ji', 976),\n",
              " ('▁Đ', 977),\n",
              " ('も', 978),\n",
              " ('▁pri', 979),\n",
              " ('▁este', 980),\n",
              " ('▁2011', 981),\n",
              " ('▁ce', 982),\n",
              " ('▁О', 983),\n",
              " ('▁է', 984),\n",
              " ('ik', 985),\n",
              " ('ት', 986),\n",
              " ('▁21', 987),\n",
              " ('는', 988),\n",
              " ('ку', 989),\n",
              " ('ж', 990),\n",
              " ('ے', 991),\n",
              " ('▁во', 992),\n",
              " ('ç', 993),\n",
              " ('ে', 994),\n",
              " ('п', 995),\n",
              " ('र', 996),\n",
              " ('Z', 997),\n",
              " ('▁од', 998),\n",
              " ('▁ob', 999),\n",
              " ...]"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "08P0YFnc3aK4"
      },
      "source": [
        "# Prepare Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xIFdSmFImJLE"
      },
      "source": [
        "# Source: https://huggingface.co/datasets/alt\n",
        "dataset = load_dataset('opus100','bn-en')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V2T0CF-Nmvgc"
      },
      "source": [
        "train_dataset = dataset['train']\n",
        "test_dataset = dataset['test']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pvow98j0kxME",
        "outputId": "4cea30d3-c658-48aa-b5a3-f006b6ec343f"
      },
      "source": [
        "train_dataset[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'translation': {'en': 'Other, Private Use', 'hi': 'अन्य, निज़ी उपयोग'}}"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ydBXAHaqPGeF"
      },
      "source": [
        "LANG_TOKEN_MAPPING = {\n",
        "\n",
        "    'bn': '<bn>',\n",
        "    'en': '<en>',\n",
        "\n",
        "}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p_a6ur1FTkD_",
        "outputId": "0320529c-8715-40b1-ac7a-0468438f3695"
      },
      "source": [
        "special_tokens_dict = {'additional_special_tokens': list(LANG_TOKEN_MAPPING.values())}\n",
        "tokenizer.add_special_tokens(special_tokens_dict)\n",
        "model.resize_token_embeddings(len(tokenizer))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Embedding(250102, 512)"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pMw-kMxyKiGC",
        "outputId": "17c811e6-3416-444c-8970-6aac1df82cc5"
      },
      "source": [
        "token_ids = tokenizer.encode(\n",
        "    example_input_str, return_tensors='pt', padding='max_length',\n",
        "    truncation=True, max_length=max_seq_len)\n",
        "print(token_ids)\n",
        "\n",
        "tokens = tokenizer.convert_ids_to_tokens(token_ids[0])\n",
        "print(tokens)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[1042, 3889,  669, 1494,  339, 1627,  259,  262, 2978,  259,  272, 1982,\n",
            "         1315,  260,    1,    0,    0,    0,    0,    0]])\n",
            "['▁<', 'jp', '>', '▁This', '▁is', '▁just', '▁', 'a', '▁test', '▁', 'n', 'bu', 'ig', '.', '</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OJNx4Mw7nlRr"
      },
      "source": [
        "def encode_input_str(text, target_lang, tokenizer, seq_len,\n",
        "                     lang_token_map=LANG_TOKEN_MAPPING):\n",
        "  target_lang_token = lang_token_map[target_lang]\n",
        "\n",
        "  # Tokenize and add special tokens\n",
        "  input_ids = tokenizer.encode(\n",
        "      text = target_lang_token + text,\n",
        "      return_tensors = 'pt',\n",
        "      padding = 'max_length',\n",
        "      truncation = True,\n",
        "      max_length = seq_len)\n",
        "\n",
        "  return input_ids[0]\n",
        "\n",
        "def encode_target_str(text, tokenizer, seq_len,\n",
        "                      lang_token_map=LANG_TOKEN_MAPPING):\n",
        "  token_ids = tokenizer.encode(\n",
        "      text = text,\n",
        "      return_tensors = 'pt',\n",
        "      padding = 'max_length',\n",
        "      truncation = True,\n",
        "      max_length = seq_len)\n",
        "\n",
        "  return token_ids[0]\n",
        "\n",
        "def format_translation_data(translations, lang_token_map,\n",
        "                            tokenizer, seq_len=128):\n",
        "  # Choose a random 2 languages for in i/o\n",
        "  langs = list(lang_token_map.keys())\n",
        "  input_lang, target_lang = np.random.choice(langs, size=2, replace=False)\n",
        "\n",
        "  # Get the translations for the batch\n",
        "  input_text = translations[input_lang]\n",
        "  target_text = translations[target_lang]\n",
        "\n",
        "  if input_text is None or target_text is None:\n",
        "    return None\n",
        "\n",
        "  input_token_ids = encode_input_str(\n",
        "      input_text, target_lang, tokenizer, seq_len, lang_token_map)\n",
        "\n",
        "  target_token_ids = encode_target_str(\n",
        "      target_text, tokenizer, seq_len, lang_token_map)\n",
        "\n",
        "  return input_token_ids, target_token_ids\n",
        "\n",
        "def transform_batch(batch, lang_token_map, tokenizer):\n",
        "  inputs = []\n",
        "  targets = []\n",
        "  for translation_set in batch['translation']:\n",
        "    formatted_data = format_translation_data(\n",
        "        translation_set, lang_token_map, tokenizer, max_seq_len)\n",
        "\n",
        "    if formatted_data is None:\n",
        "      continue\n",
        "\n",
        "    input_ids, target_ids = formatted_data\n",
        "    inputs.append(input_ids.unsqueeze(0))\n",
        "    targets.append(target_ids.unsqueeze(0))\n",
        "\n",
        "  batch_input_ids = torch.cat(inputs).cuda()\n",
        "  batch_target_ids = torch.cat(targets).cuda()\n",
        "\n",
        "  return batch_input_ids, batch_target_ids\n",
        "\n",
        "def get_data_generator(dataset, lang_token_map, tokenizer, batch_size=32):\n",
        "  dataset = dataset.shuffle()\n",
        "  for i in range(0, len(dataset), batch_size):\n",
        "    raw_batch = dataset[i:i+batch_size]\n",
        "    yield transform_batch(raw_batch, lang_token_map, tokenizer)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jmdUeqObF_j_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ab06876e-1be6-44c7-f77d-aff11faa1b4e"
      },
      "source": [
        "# Testing `data_transform`\n",
        "in_ids, out_ids = format_translation_data(\n",
        "    train_dataset[0]['translation'], LANG_TOKEN_MAPPING, tokenizer)\n",
        "\n",
        "print(' '.join(tokenizer.convert_ids_to_tokens(in_ids)))\n",
        "print(' '.join(tokenizer.convert_ids_to_tokens(out_ids)))\n",
        "\n",
        "# Testing data generator\n",
        "data_gen = get_data_generator(train_dataset, LANG_TOKEN_MAPPING, tokenizer, 8)\n",
        "data_batch = next(data_gen)\n",
        "print('Input shape:', data_batch[0].shape)\n",
        "print('Output shape:', data_batch[1].shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<hi> ▁Other , ▁Private ▁Use </s> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
            "▁अ न्य , ▁नि ज़ ी ▁उप योग </s> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
            "Input shape: torch.Size([8, 20])\n",
            "Output shape: torch.Size([8, 20])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xKLEqcU4m9M0"
      },
      "source": [
        "# Training/Finetune google/mt5-small model\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-4uv5u_FnE2F"
      },
      "source": [
        "# Constants\n",
        "n_epochs = 1\n",
        "batch_size = 16\n",
        "print_freq = 50\n",
        "checkpoint_freq = 1000\n",
        "lr = 5e-4\n",
        "n_batches = int(np.ceil(len(train_dataset) / batch_size))\n",
        "total_steps = n_epochs * n_batches\n",
        "n_warmup_steps = int(total_steps * 0.01)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yv-85lPx-Oo7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1c747c32-e675-4909-95e0-6de6ed39d4f3"
      },
      "source": [
        "# Optimizer\n",
        "optimizer = AdamW(model.parameters(), lr=lr)\n",
        "scheduler = get_linear_schedule_with_warmup(\n",
        "    optimizer, n_warmup_steps, total_steps)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OLN37ltk__ws"
      },
      "source": [
        "losses = []"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rKU9rtbHWkeB"
      },
      "source": [
        "def eval_model(model, gdataset, max_iters=8):\n",
        "  test_generator = get_data_generator(gdataset, LANG_TOKEN_MAPPING,\n",
        "                                      tokenizer, batch_size)\n",
        "  eval_losses = []\n",
        "  for i, (input_batch, label_batch) in enumerate(test_generator):\n",
        "    if i >= max_iters:\n",
        "      break\n",
        "\n",
        "    model_out = model.forward(\n",
        "        input_ids = input_batch,\n",
        "        labels = label_batch)\n",
        "    eval_losses.append(model_out.loss.item())\n",
        "\n",
        "  return np.mean(eval_losses)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "473dd75b8f9c46d997e7136e614c2d8c",
            "9feab48c67d24f948075684a2a26a217",
            "3560154ac4a84b1ab56465c80c753538",
            "3259a3cad58c407daca409ee3d321e6d",
            "23a90dcf563a40fa8480166377c510f5",
            "fcd19ad63bb34b30b04b09f445222e96",
            "0ffcf02892534d6593ae806c62a1de0b",
            "28461c2fcb8b4359a4201858342da740",
            "878748c12a5b429e986ea516932f2b3f",
            "6ffcdcaefbd44ba594b84c6e9c178edc",
            "6f34900b4c4844eaa2515d7f559b5f1f"
          ]
        },
        "id": "Kv8a0jwDnEzK",
        "outputId": "84d5bdc9-6fb0-4c20-ff82-2cc6663f13dc"
      },
      "source": [
        "for epoch_idx in range(n_epochs):\n",
        "  # Randomize data order\n",
        "  data_generator = get_data_generator(train_dataset, LANG_TOKEN_MAPPING,\n",
        "                                      tokenizer, batch_size)\n",
        "\n",
        "  for batch_idx, (input_batch, label_batch) \\\n",
        "      in tqdm_notebook(enumerate(data_generator), total=n_batches):\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    # Forward pass\n",
        "    model_out = model.forward(\n",
        "        input_ids = input_batch,\n",
        "        labels = label_batch)\n",
        "\n",
        "    # Calculate loss and update weights\n",
        "    loss = model_out.loss\n",
        "    losses.append(loss.item())\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    scheduler.step()\n",
        "\n",
        "    # Print training update info\n",
        "    if (batch_idx + 1) % print_freq == 0:\n",
        "      avg_loss = np.mean(losses[-print_freq:])\n",
        "      print('Epoch: {} | Step: {} | Avg. loss: {:.3f} | lr: {}'.format(\n",
        "          epoch_idx+1, batch_idx+1, avg_loss, scheduler.get_last_lr()[0]))\n",
        "\n",
        "    if (batch_idx + 1) % checkpoint_freq == 0:\n",
        "      test_loss = eval_model(model, test_dataset)\n",
        "      print('Saving model with test loss of {:.3f}'.format(test_loss))\n",
        "      torch.save(model.state_dict(), model_path)\n",
        "\n",
        "torch.save(model.state_dict(), model_path)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-22-9682a3c86c07>:7: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
            "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
            "  in tqdm_notebook(enumerate(data_generator), total=n_batches):\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/33395 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "473dd75b8f9c46d997e7136e614c2d8c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 1 | Step: 50 | Avg. loss: 35.510 | lr: 7.507507507507507e-05\n",
            "Epoch: 1 | Step: 100 | Avg. loss: 19.304 | lr: 0.00015015015015015014\n",
            "Epoch: 1 | Step: 150 | Avg. loss: 10.204 | lr: 0.00022522522522522523\n",
            "Epoch: 1 | Step: 200 | Avg. loss: 7.310 | lr: 0.0003003003003003003\n",
            "Epoch: 1 | Step: 250 | Avg. loss: 4.788 | lr: 0.00037537537537537537\n",
            "Epoch: 1 | Step: 300 | Avg. loss: 3.445 | lr: 0.00045045045045045046\n",
            "Epoch: 1 | Step: 350 | Avg. loss: 3.184 | lr: 0.0004997429072651382\n",
            "Epoch: 1 | Step: 400 | Avg. loss: 3.027 | lr: 0.0004989867521626036\n",
            "Epoch: 1 | Step: 450 | Avg. loss: 2.697 | lr: 0.0004982305970600689\n",
            "Epoch: 1 | Step: 500 | Avg. loss: 2.518 | lr: 0.0004974744419575343\n",
            "Epoch: 1 | Step: 550 | Avg. loss: 2.421 | lr: 0.0004967182868549997\n",
            "Epoch: 1 | Step: 600 | Avg. loss: 2.277 | lr: 0.000495962131752465\n",
            "Epoch: 1 | Step: 650 | Avg. loss: 2.320 | lr: 0.0004952059766499305\n",
            "Epoch: 1 | Step: 700 | Avg. loss: 2.266 | lr: 0.0004944498215473959\n",
            "Epoch: 1 | Step: 750 | Avg. loss: 2.156 | lr: 0.0004936936664448612\n",
            "Epoch: 1 | Step: 800 | Avg. loss: 2.134 | lr: 0.0004929375113423266\n",
            "Epoch: 1 | Step: 850 | Avg. loss: 2.161 | lr: 0.000492181356239792\n",
            "Epoch: 1 | Step: 900 | Avg. loss: 2.136 | lr: 0.0004914252011372573\n",
            "Epoch: 1 | Step: 950 | Avg. loss: 2.122 | lr: 0.0004906690460347227\n",
            "Epoch: 1 | Step: 1000 | Avg. loss: 2.093 | lr: 0.000489912890932188\n",
            "Saving model with test loss of 2.087\n",
            "Epoch: 1 | Step: 1050 | Avg. loss: 2.038 | lr: 0.0004891567358296533\n",
            "Epoch: 1 | Step: 1100 | Avg. loss: 1.980 | lr: 0.0004884005807271188\n",
            "Epoch: 1 | Step: 1150 | Avg. loss: 1.998 | lr: 0.0004876444256245841\n",
            "Epoch: 1 | Step: 1200 | Avg. loss: 2.007 | lr: 0.00048688827052204947\n",
            "Epoch: 1 | Step: 1250 | Avg. loss: 2.001 | lr: 0.00048613211541951483\n",
            "Epoch: 1 | Step: 1300 | Avg. loss: 1.976 | lr: 0.0004853759603169802\n",
            "Epoch: 1 | Step: 1350 | Avg. loss: 1.922 | lr: 0.0004846198052144456\n",
            "Epoch: 1 | Step: 1400 | Avg. loss: 1.928 | lr: 0.000483863650111911\n",
            "Epoch: 1 | Step: 1450 | Avg. loss: 1.853 | lr: 0.00048310749500937635\n",
            "Epoch: 1 | Step: 1500 | Avg. loss: 1.967 | lr: 0.0004823513399068417\n",
            "Epoch: 1 | Step: 1550 | Avg. loss: 1.912 | lr: 0.0004815951848043071\n",
            "Epoch: 1 | Step: 1600 | Avg. loss: 1.860 | lr: 0.00048083902970177245\n",
            "Epoch: 1 | Step: 1650 | Avg. loss: 1.760 | lr: 0.0004800828745992378\n",
            "Epoch: 1 | Step: 1700 | Avg. loss: 1.866 | lr: 0.0004793267194967032\n",
            "Epoch: 1 | Step: 1750 | Avg. loss: 1.888 | lr: 0.0004785705643941685\n",
            "Epoch: 1 | Step: 1800 | Avg. loss: 1.849 | lr: 0.0004778144092916339\n",
            "Epoch: 1 | Step: 1850 | Avg. loss: 1.807 | lr: 0.0004770582541890993\n",
            "Epoch: 1 | Step: 1900 | Avg. loss: 1.839 | lr: 0.00047630209908656464\n",
            "Epoch: 1 | Step: 1950 | Avg. loss: 1.758 | lr: 0.00047554594398403\n",
            "Epoch: 1 | Step: 2000 | Avg. loss: 1.806 | lr: 0.0004747897888814954\n",
            "Saving model with test loss of 1.909\n",
            "Epoch: 1 | Step: 2050 | Avg. loss: 1.774 | lr: 0.00047403363377896074\n",
            "Epoch: 1 | Step: 2100 | Avg. loss: 1.850 | lr: 0.0004732774786764261\n",
            "Epoch: 1 | Step: 2150 | Avg. loss: 1.761 | lr: 0.0004725213235738915\n",
            "Epoch: 1 | Step: 2200 | Avg. loss: 1.762 | lr: 0.00047176516847135684\n",
            "Epoch: 1 | Step: 2250 | Avg. loss: 1.787 | lr: 0.00047100901336882226\n",
            "Epoch: 1 | Step: 2300 | Avg. loss: 1.791 | lr: 0.0004702528582662876\n",
            "Epoch: 1 | Step: 2350 | Avg. loss: 1.725 | lr: 0.000469496703163753\n",
            "Epoch: 1 | Step: 2400 | Avg. loss: 1.747 | lr: 0.00046874054806121836\n",
            "Epoch: 1 | Step: 2450 | Avg. loss: 1.757 | lr: 0.00046798439295868367\n",
            "Epoch: 1 | Step: 2500 | Avg. loss: 1.719 | lr: 0.00046722823785614904\n",
            "Epoch: 1 | Step: 2550 | Avg. loss: 1.719 | lr: 0.0004664720827536144\n",
            "Epoch: 1 | Step: 2600 | Avg. loss: 1.699 | lr: 0.00046571592765107977\n",
            "Epoch: 1 | Step: 2650 | Avg. loss: 1.681 | lr: 0.00046495977254854513\n",
            "Epoch: 1 | Step: 2700 | Avg. loss: 1.749 | lr: 0.00046420361744601055\n",
            "Epoch: 1 | Step: 2750 | Avg. loss: 1.683 | lr: 0.0004634474623434759\n",
            "Epoch: 1 | Step: 2800 | Avg. loss: 1.735 | lr: 0.0004626913072409413\n",
            "Epoch: 1 | Step: 2850 | Avg. loss: 1.693 | lr: 0.00046193515213840665\n",
            "Epoch: 1 | Step: 2900 | Avg. loss: 1.677 | lr: 0.000461178997035872\n",
            "Epoch: 1 | Step: 2950 | Avg. loss: 1.728 | lr: 0.0004604228419333374\n",
            "Epoch: 1 | Step: 3000 | Avg. loss: 1.669 | lr: 0.00045966668683080275\n",
            "Saving model with test loss of 1.701\n",
            "Epoch: 1 | Step: 3050 | Avg. loss: 1.704 | lr: 0.0004589105317282681\n",
            "Epoch: 1 | Step: 3100 | Avg. loss: 1.672 | lr: 0.0004581543766257335\n",
            "Epoch: 1 | Step: 3150 | Avg. loss: 1.671 | lr: 0.00045739822152319885\n",
            "Epoch: 1 | Step: 3200 | Avg. loss: 1.619 | lr: 0.0004566420664206642\n",
            "Epoch: 1 | Step: 3250 | Avg. loss: 1.621 | lr: 0.0004558859113181296\n",
            "Epoch: 1 | Step: 3300 | Avg. loss: 1.637 | lr: 0.00045512975621559494\n",
            "Epoch: 1 | Step: 3350 | Avg. loss: 1.636 | lr: 0.0004543736011130603\n",
            "Epoch: 1 | Step: 3400 | Avg. loss: 1.582 | lr: 0.0004536174460105257\n",
            "Epoch: 1 | Step: 3450 | Avg. loss: 1.670 | lr: 0.00045286129090799104\n",
            "Epoch: 1 | Step: 3500 | Avg. loss: 1.643 | lr: 0.0004521051358054564\n",
            "Epoch: 1 | Step: 3550 | Avg. loss: 1.627 | lr: 0.0004513489807029218\n",
            "Epoch: 1 | Step: 3600 | Avg. loss: 1.573 | lr: 0.0004505928256003872\n",
            "Epoch: 1 | Step: 3650 | Avg. loss: 1.528 | lr: 0.00044983667049785256\n",
            "Epoch: 1 | Step: 3700 | Avg. loss: 1.609 | lr: 0.0004490805153953179\n",
            "Epoch: 1 | Step: 3750 | Avg. loss: 1.592 | lr: 0.0004483243602927833\n",
            "Epoch: 1 | Step: 3800 | Avg. loss: 1.559 | lr: 0.00044756820519024866\n",
            "Epoch: 1 | Step: 3850 | Avg. loss: 1.600 | lr: 0.000446812050087714\n",
            "Epoch: 1 | Step: 3900 | Avg. loss: 1.586 | lr: 0.00044605589498517933\n",
            "Epoch: 1 | Step: 3950 | Avg. loss: 1.567 | lr: 0.0004452997398826447\n",
            "Epoch: 1 | Step: 4000 | Avg. loss: 1.579 | lr: 0.00044454358478011007\n",
            "Saving model with test loss of 1.658\n",
            "Epoch: 1 | Step: 4050 | Avg. loss: 1.588 | lr: 0.0004437874296775755\n",
            "Epoch: 1 | Step: 4100 | Avg. loss: 1.653 | lr: 0.00044303127457504085\n",
            "Epoch: 1 | Step: 4150 | Avg. loss: 1.503 | lr: 0.0004422751194725062\n",
            "Epoch: 1 | Step: 4200 | Avg. loss: 1.511 | lr: 0.0004415189643699716\n",
            "Epoch: 1 | Step: 4250 | Avg. loss: 1.670 | lr: 0.00044076280926743695\n",
            "Epoch: 1 | Step: 4300 | Avg. loss: 1.607 | lr: 0.0004400066541649023\n",
            "Epoch: 1 | Step: 4350 | Avg. loss: 1.553 | lr: 0.0004392504990623677\n",
            "Epoch: 1 | Step: 4400 | Avg. loss: 1.547 | lr: 0.00043849434395983305\n",
            "Epoch: 1 | Step: 4450 | Avg. loss: 1.525 | lr: 0.0004377381888572984\n",
            "Epoch: 1 | Step: 4500 | Avg. loss: 1.528 | lr: 0.00043698203375476383\n",
            "Epoch: 1 | Step: 4550 | Avg. loss: 1.570 | lr: 0.0004362258786522292\n",
            "Epoch: 1 | Step: 4600 | Avg. loss: 1.544 | lr: 0.0004354697235496945\n",
            "Epoch: 1 | Step: 4650 | Avg. loss: 1.533 | lr: 0.0004347135684471599\n",
            "Epoch: 1 | Step: 4700 | Avg. loss: 1.546 | lr: 0.00043395741334462524\n",
            "Epoch: 1 | Step: 4750 | Avg. loss: 1.528 | lr: 0.0004332012582420906\n",
            "Epoch: 1 | Step: 4800 | Avg. loss: 1.439 | lr: 0.000432445103139556\n",
            "Epoch: 1 | Step: 4850 | Avg. loss: 1.512 | lr: 0.00043168894803702134\n",
            "Epoch: 1 | Step: 4900 | Avg. loss: 1.474 | lr: 0.0004309327929344867\n",
            "Epoch: 1 | Step: 4950 | Avg. loss: 1.494 | lr: 0.00043017663783195213\n",
            "Epoch: 1 | Step: 5000 | Avg. loss: 1.491 | lr: 0.0004294204827294175\n",
            "Saving model with test loss of 1.697\n",
            "Epoch: 1 | Step: 5050 | Avg. loss: 1.516 | lr: 0.00042866432762688286\n",
            "Epoch: 1 | Step: 5100 | Avg. loss: 1.485 | lr: 0.0004279081725243482\n",
            "Epoch: 1 | Step: 5150 | Avg. loss: 1.515 | lr: 0.0004271520174218136\n",
            "Epoch: 1 | Step: 5200 | Avg. loss: 1.543 | lr: 0.00042639586231927896\n",
            "Epoch: 1 | Step: 5250 | Avg. loss: 1.419 | lr: 0.0004256397072167443\n",
            "Epoch: 1 | Step: 5300 | Avg. loss: 1.499 | lr: 0.00042488355211420963\n",
            "Epoch: 1 | Step: 5350 | Avg. loss: 1.474 | lr: 0.000424127397011675\n",
            "Epoch: 1 | Step: 5400 | Avg. loss: 1.436 | lr: 0.00042337124190914037\n",
            "Epoch: 1 | Step: 5450 | Avg. loss: 1.514 | lr: 0.0004226150868066058\n",
            "Epoch: 1 | Step: 5500 | Avg. loss: 1.426 | lr: 0.00042185893170407115\n",
            "Epoch: 1 | Step: 5550 | Avg. loss: 1.488 | lr: 0.0004211027766015365\n",
            "Epoch: 1 | Step: 5600 | Avg. loss: 1.425 | lr: 0.0004203466214990019\n",
            "Epoch: 1 | Step: 5650 | Avg. loss: 1.431 | lr: 0.00041959046639646725\n",
            "Epoch: 1 | Step: 5700 | Avg. loss: 1.472 | lr: 0.0004188343112939326\n",
            "Epoch: 1 | Step: 5750 | Avg. loss: 1.485 | lr: 0.000418078156191398\n",
            "Epoch: 1 | Step: 5800 | Avg. loss: 1.470 | lr: 0.00041732200108886335\n",
            "Epoch: 1 | Step: 5850 | Avg. loss: 1.476 | lr: 0.0004165658459863287\n",
            "Epoch: 1 | Step: 5900 | Avg. loss: 1.508 | lr: 0.00041580969088379413\n",
            "Epoch: 1 | Step: 5950 | Avg. loss: 1.418 | lr: 0.0004150535357812595\n",
            "Epoch: 1 | Step: 6000 | Avg. loss: 1.468 | lr: 0.00041429738067872487\n",
            "Saving model with test loss of 1.555\n",
            "Epoch: 1 | Step: 6050 | Avg. loss: 1.380 | lr: 0.0004135412255761902\n",
            "Epoch: 1 | Step: 6100 | Avg. loss: 1.395 | lr: 0.00041278507047365554\n",
            "Epoch: 1 | Step: 6150 | Avg. loss: 1.427 | lr: 0.0004120289153711209\n",
            "Epoch: 1 | Step: 6200 | Avg. loss: 1.437 | lr: 0.0004112727602685863\n",
            "Epoch: 1 | Step: 6250 | Avg. loss: 1.449 | lr: 0.00041051660516605164\n",
            "Epoch: 1 | Step: 6300 | Avg. loss: 1.446 | lr: 0.000409760450063517\n",
            "Epoch: 1 | Step: 6350 | Avg. loss: 1.458 | lr: 0.0004090042949609824\n",
            "Epoch: 1 | Step: 6400 | Avg. loss: 1.429 | lr: 0.0004082481398584478\n",
            "Epoch: 1 | Step: 6450 | Avg. loss: 1.379 | lr: 0.00040749198475591316\n",
            "Epoch: 1 | Step: 6500 | Avg. loss: 1.494 | lr: 0.0004067358296533785\n",
            "Epoch: 1 | Step: 6550 | Avg. loss: 1.397 | lr: 0.0004059796745508439\n",
            "Epoch: 1 | Step: 6600 | Avg. loss: 1.498 | lr: 0.00040522351944830926\n",
            "Epoch: 1 | Step: 6650 | Avg. loss: 1.437 | lr: 0.0004044673643457746\n",
            "Epoch: 1 | Step: 6700 | Avg. loss: 1.422 | lr: 0.00040371120924324\n",
            "Epoch: 1 | Step: 6750 | Avg. loss: 1.467 | lr: 0.0004029550541407053\n",
            "Epoch: 1 | Step: 6800 | Avg. loss: 1.416 | lr: 0.0004021988990381707\n",
            "Epoch: 1 | Step: 6850 | Avg. loss: 1.393 | lr: 0.0004014427439356361\n",
            "Epoch: 1 | Step: 6900 | Avg. loss: 1.390 | lr: 0.00040068658883310145\n",
            "Epoch: 1 | Step: 6950 | Avg. loss: 1.393 | lr: 0.0003999304337305668\n",
            "Epoch: 1 | Step: 7000 | Avg. loss: 1.405 | lr: 0.0003991742786280322\n",
            "Saving model with test loss of 1.662\n",
            "Epoch: 1 | Step: 7050 | Avg. loss: 1.439 | lr: 0.00039841812352549755\n",
            "Epoch: 1 | Step: 7100 | Avg. loss: 1.410 | lr: 0.0003976619684229629\n",
            "Epoch: 1 | Step: 7150 | Avg. loss: 1.406 | lr: 0.0003969058133204283\n",
            "Epoch: 1 | Step: 7200 | Avg. loss: 1.388 | lr: 0.00039614965821789365\n",
            "Epoch: 1 | Step: 7250 | Avg. loss: 1.405 | lr: 0.00039539350311535907\n",
            "Epoch: 1 | Step: 7300 | Avg. loss: 1.385 | lr: 0.00039463734801282443\n",
            "Epoch: 1 | Step: 7350 | Avg. loss: 1.352 | lr: 0.0003938811929102898\n",
            "Epoch: 1 | Step: 7400 | Avg. loss: 1.380 | lr: 0.00039312503780775516\n",
            "Epoch: 1 | Step: 7450 | Avg. loss: 1.385 | lr: 0.0003923688827052205\n",
            "Epoch: 1 | Step: 7500 | Avg. loss: 1.366 | lr: 0.00039161272760268584\n",
            "Epoch: 1 | Step: 7550 | Avg. loss: 1.396 | lr: 0.0003908565725001512\n",
            "Epoch: 1 | Step: 7600 | Avg. loss: 1.385 | lr: 0.0003901004173976166\n",
            "Epoch: 1 | Step: 7650 | Avg. loss: 1.383 | lr: 0.00038934426229508194\n",
            "Epoch: 1 | Step: 7700 | Avg. loss: 1.394 | lr: 0.00038858810719254736\n",
            "Epoch: 1 | Step: 7750 | Avg. loss: 1.397 | lr: 0.0003878319520900127\n",
            "Epoch: 1 | Step: 7800 | Avg. loss: 1.449 | lr: 0.0003870757969874781\n",
            "Epoch: 1 | Step: 7850 | Avg. loss: 1.387 | lr: 0.00038631964188494346\n",
            "Epoch: 1 | Step: 7900 | Avg. loss: 1.365 | lr: 0.0003855634867824088\n",
            "Epoch: 1 | Step: 7950 | Avg. loss: 1.395 | lr: 0.0003848073316798742\n",
            "Epoch: 1 | Step: 8000 | Avg. loss: 1.323 | lr: 0.00038405117657733956\n",
            "Saving model with test loss of 1.636\n",
            "Epoch: 1 | Step: 8050 | Avg. loss: 1.355 | lr: 0.0003832950214748049\n",
            "Epoch: 1 | Step: 8100 | Avg. loss: 1.355 | lr: 0.0003825388663722703\n",
            "Epoch: 1 | Step: 8150 | Avg. loss: 1.405 | lr: 0.0003817827112697357\n",
            "Epoch: 1 | Step: 8200 | Avg. loss: 1.343 | lr: 0.000381026556167201\n",
            "Epoch: 1 | Step: 8250 | Avg. loss: 1.382 | lr: 0.0003802704010646664\n",
            "Epoch: 1 | Step: 8300 | Avg. loss: 1.392 | lr: 0.00037951424596213175\n",
            "Epoch: 1 | Step: 8350 | Avg. loss: 1.358 | lr: 0.0003787580908595971\n",
            "Epoch: 1 | Step: 8400 | Avg. loss: 1.359 | lr: 0.0003780019357570625\n",
            "Epoch: 1 | Step: 8450 | Avg. loss: 1.374 | lr: 0.00037724578065452785\n",
            "Epoch: 1 | Step: 8500 | Avg. loss: 1.354 | lr: 0.0003764896255519932\n",
            "Epoch: 1 | Step: 8550 | Avg. loss: 1.330 | lr: 0.0003757334704494586\n",
            "Epoch: 1 | Step: 8600 | Avg. loss: 1.226 | lr: 0.000374977315346924\n",
            "Epoch: 1 | Step: 8650 | Avg. loss: 1.356 | lr: 0.00037422116024438937\n",
            "Epoch: 1 | Step: 8700 | Avg. loss: 1.382 | lr: 0.00037346500514185473\n",
            "Epoch: 1 | Step: 8750 | Avg. loss: 1.396 | lr: 0.0003727088500393201\n",
            "Epoch: 1 | Step: 8800 | Avg. loss: 1.352 | lr: 0.00037195269493678546\n",
            "Epoch: 1 | Step: 8850 | Avg. loss: 1.347 | lr: 0.00037119653983425083\n",
            "Epoch: 1 | Step: 8900 | Avg. loss: 1.290 | lr: 0.00037044038473171614\n",
            "Epoch: 1 | Step: 8950 | Avg. loss: 1.342 | lr: 0.0003696842296291815\n",
            "Epoch: 1 | Step: 9000 | Avg. loss: 1.306 | lr: 0.0003689280745266469\n",
            "Saving model with test loss of 1.544\n",
            "Epoch: 1 | Step: 9050 | Avg. loss: 1.342 | lr: 0.0003681719194241123\n",
            "Epoch: 1 | Step: 9100 | Avg. loss: 1.361 | lr: 0.00036741576432157766\n",
            "Epoch: 1 | Step: 9150 | Avg. loss: 1.346 | lr: 0.000366659609219043\n",
            "Epoch: 1 | Step: 9200 | Avg. loss: 1.289 | lr: 0.0003659034541165084\n",
            "Epoch: 1 | Step: 9250 | Avg. loss: 1.315 | lr: 0.00036514729901397376\n",
            "Epoch: 1 | Step: 9300 | Avg. loss: 1.275 | lr: 0.0003643911439114391\n",
            "Epoch: 1 | Step: 9350 | Avg. loss: 1.306 | lr: 0.0003636349888089045\n",
            "Epoch: 1 | Step: 9400 | Avg. loss: 1.281 | lr: 0.00036287883370636985\n",
            "Epoch: 1 | Step: 9450 | Avg. loss: 1.346 | lr: 0.0003621226786038352\n",
            "Epoch: 1 | Step: 9500 | Avg. loss: 1.365 | lr: 0.00036136652350130064\n",
            "Epoch: 1 | Step: 9550 | Avg. loss: 1.337 | lr: 0.000360610368398766\n",
            "Epoch: 1 | Step: 9600 | Avg. loss: 1.343 | lr: 0.0003598542132962313\n",
            "Epoch: 1 | Step: 9650 | Avg. loss: 1.362 | lr: 0.0003590980581936967\n",
            "Epoch: 1 | Step: 9700 | Avg. loss: 1.288 | lr: 0.00035834190309116205\n",
            "Epoch: 1 | Step: 9750 | Avg. loss: 1.278 | lr: 0.0003575857479886274\n",
            "Epoch: 1 | Step: 9800 | Avg. loss: 1.327 | lr: 0.0003568295928860928\n",
            "Epoch: 1 | Step: 9850 | Avg. loss: 1.326 | lr: 0.00035607343778355815\n",
            "Epoch: 1 | Step: 9900 | Avg. loss: 1.318 | lr: 0.0003553172826810235\n",
            "Epoch: 1 | Step: 9950 | Avg. loss: 1.358 | lr: 0.00035456112757848893\n",
            "Epoch: 1 | Step: 10000 | Avg. loss: 1.258 | lr: 0.0003538049724759543\n",
            "Saving model with test loss of 1.449\n",
            "Epoch: 1 | Step: 10050 | Avg. loss: 1.274 | lr: 0.00035304881737341967\n",
            "Epoch: 1 | Step: 10100 | Avg. loss: 1.321 | lr: 0.00035229266227088503\n",
            "Epoch: 1 | Step: 10150 | Avg. loss: 1.249 | lr: 0.0003515365071683504\n",
            "Epoch: 1 | Step: 10200 | Avg. loss: 1.280 | lr: 0.00035078035206581576\n",
            "Epoch: 1 | Step: 10250 | Avg. loss: 1.288 | lr: 0.00035002419696328113\n",
            "Epoch: 1 | Step: 10300 | Avg. loss: 1.260 | lr: 0.0003492680418607465\n",
            "Epoch: 1 | Step: 10350 | Avg. loss: 1.302 | lr: 0.0003485118867582118\n",
            "Epoch: 1 | Step: 10400 | Avg. loss: 1.337 | lr: 0.00034775573165567717\n",
            "Epoch: 1 | Step: 10450 | Avg. loss: 1.275 | lr: 0.0003469995765531426\n",
            "Epoch: 1 | Step: 10500 | Avg. loss: 1.339 | lr: 0.00034624342145060796\n",
            "Epoch: 1 | Step: 10550 | Avg. loss: 1.332 | lr: 0.0003454872663480733\n",
            "Epoch: 1 | Step: 10600 | Avg. loss: 1.273 | lr: 0.0003447311112455387\n",
            "Epoch: 1 | Step: 10650 | Avg. loss: 1.287 | lr: 0.00034397495614300406\n",
            "Epoch: 1 | Step: 10700 | Avg. loss: 1.264 | lr: 0.0003432188010404694\n",
            "Epoch: 1 | Step: 10750 | Avg. loss: 1.297 | lr: 0.0003424626459379348\n",
            "Epoch: 1 | Step: 10800 | Avg. loss: 1.257 | lr: 0.00034170649083540015\n",
            "Epoch: 1 | Step: 10850 | Avg. loss: 1.239 | lr: 0.0003409503357328655\n",
            "Epoch: 1 | Step: 10900 | Avg. loss: 1.264 | lr: 0.00034019418063033094\n",
            "Epoch: 1 | Step: 10950 | Avg. loss: 1.322 | lr: 0.0003394380255277963\n",
            "Epoch: 1 | Step: 11000 | Avg. loss: 1.297 | lr: 0.00033868187042526167\n",
            "Saving model with test loss of 1.421\n",
            "Epoch: 1 | Step: 11050 | Avg. loss: 1.305 | lr: 0.000337925715322727\n",
            "Epoch: 1 | Step: 11100 | Avg. loss: 1.279 | lr: 0.00033716956022019235\n",
            "Epoch: 1 | Step: 11150 | Avg. loss: 1.280 | lr: 0.0003364134051176577\n",
            "Epoch: 1 | Step: 11200 | Avg. loss: 1.321 | lr: 0.0003356572500151231\n",
            "Epoch: 1 | Step: 11250 | Avg. loss: 1.266 | lr: 0.00033490109491258845\n",
            "Epoch: 1 | Step: 11300 | Avg. loss: 1.252 | lr: 0.0003341449398100538\n",
            "Epoch: 1 | Step: 11350 | Avg. loss: 1.274 | lr: 0.00033338878470751923\n",
            "Epoch: 1 | Step: 11400 | Avg. loss: 1.292 | lr: 0.0003326326296049846\n",
            "Epoch: 1 | Step: 11450 | Avg. loss: 1.187 | lr: 0.00033187647450244996\n",
            "Epoch: 1 | Step: 11500 | Avg. loss: 1.214 | lr: 0.00033112031939991533\n",
            "Epoch: 1 | Step: 11550 | Avg. loss: 1.261 | lr: 0.0003303641642973807\n",
            "Epoch: 1 | Step: 11600 | Avg. loss: 1.273 | lr: 0.00032960800919484606\n",
            "Epoch: 1 | Step: 11650 | Avg. loss: 1.275 | lr: 0.00032885185409231143\n",
            "Epoch: 1 | Step: 11700 | Avg. loss: 1.271 | lr: 0.0003280956989897768\n",
            "Epoch: 1 | Step: 11750 | Avg. loss: 1.293 | lr: 0.0003273395438872421\n",
            "Epoch: 1 | Step: 11800 | Avg. loss: 1.299 | lr: 0.0003265833887847075\n",
            "Epoch: 1 | Step: 11850 | Avg. loss: 1.247 | lr: 0.0003258272336821729\n",
            "Epoch: 1 | Step: 11900 | Avg. loss: 1.265 | lr: 0.00032507107857963826\n",
            "Epoch: 1 | Step: 11950 | Avg. loss: 1.289 | lr: 0.0003243149234771036\n",
            "Epoch: 1 | Step: 12000 | Avg. loss: 1.265 | lr: 0.000323558768374569\n",
            "Saving model with test loss of 1.451\n",
            "Epoch: 1 | Step: 12050 | Avg. loss: 1.267 | lr: 0.00032280261327203436\n",
            "Epoch: 1 | Step: 12100 | Avg. loss: 1.238 | lr: 0.0003220464581694997\n",
            "Epoch: 1 | Step: 12150 | Avg. loss: 1.205 | lr: 0.0003212903030669651\n",
            "Epoch: 1 | Step: 12200 | Avg. loss: 1.224 | lr: 0.00032053414796443045\n",
            "Epoch: 1 | Step: 12250 | Avg. loss: 1.236 | lr: 0.0003197779928618959\n",
            "Epoch: 1 | Step: 12300 | Avg. loss: 1.279 | lr: 0.00031902183775936124\n",
            "Epoch: 1 | Step: 12350 | Avg. loss: 1.255 | lr: 0.0003182656826568266\n",
            "Epoch: 1 | Step: 12400 | Avg. loss: 1.210 | lr: 0.00031750952755429197\n",
            "Epoch: 1 | Step: 12450 | Avg. loss: 1.320 | lr: 0.00031675337245175734\n",
            "Epoch: 1 | Step: 12500 | Avg. loss: 1.255 | lr: 0.00031599721734922265\n",
            "Epoch: 1 | Step: 12550 | Avg. loss: 1.178 | lr: 0.000315241062246688\n",
            "Epoch: 1 | Step: 12600 | Avg. loss: 1.226 | lr: 0.0003144849071441534\n",
            "Epoch: 1 | Step: 12650 | Avg. loss: 1.240 | lr: 0.00031372875204161875\n",
            "Epoch: 1 | Step: 12700 | Avg. loss: 1.218 | lr: 0.00031297259693908417\n",
            "Epoch: 1 | Step: 12750 | Avg. loss: 1.201 | lr: 0.00031221644183654953\n",
            "Epoch: 1 | Step: 12800 | Avg. loss: 1.164 | lr: 0.0003114602867340149\n",
            "Epoch: 1 | Step: 12850 | Avg. loss: 1.201 | lr: 0.00031070413163148026\n",
            "Epoch: 1 | Step: 12900 | Avg. loss: 1.193 | lr: 0.00030994797652894563\n",
            "Epoch: 1 | Step: 12950 | Avg. loss: 1.163 | lr: 0.000309191821426411\n",
            "Epoch: 1 | Step: 13000 | Avg. loss: 1.237 | lr: 0.00030843566632387636\n",
            "Saving model with test loss of 1.549\n",
            "Epoch: 1 | Step: 13050 | Avg. loss: 1.266 | lr: 0.00030767951122134173\n",
            "Epoch: 1 | Step: 13100 | Avg. loss: 1.242 | lr: 0.0003069233561188071\n",
            "Epoch: 1 | Step: 13150 | Avg. loss: 1.266 | lr: 0.0003061672010162725\n",
            "Epoch: 1 | Step: 13200 | Avg. loss: 1.224 | lr: 0.0003054110459137378\n",
            "Epoch: 1 | Step: 13250 | Avg. loss: 1.255 | lr: 0.0003046548908112032\n",
            "Epoch: 1 | Step: 13300 | Avg. loss: 1.171 | lr: 0.00030389873570866856\n",
            "Epoch: 1 | Step: 13350 | Avg. loss: 1.193 | lr: 0.0003031425806061339\n",
            "Epoch: 1 | Step: 13400 | Avg. loss: 1.210 | lr: 0.0003023864255035993\n",
            "Epoch: 1 | Step: 13450 | Avg. loss: 1.281 | lr: 0.00030163027040106465\n",
            "Epoch: 1 | Step: 13500 | Avg. loss: 1.206 | lr: 0.00030087411529853\n",
            "Epoch: 1 | Step: 13550 | Avg. loss: 1.209 | lr: 0.0003001179601959954\n",
            "Epoch: 1 | Step: 13600 | Avg. loss: 1.191 | lr: 0.0002993618050934608\n",
            "Epoch: 1 | Step: 13650 | Avg. loss: 1.260 | lr: 0.00029860564999092617\n",
            "Epoch: 1 | Step: 13700 | Avg. loss: 1.215 | lr: 0.00029784949488839154\n",
            "Epoch: 1 | Step: 13750 | Avg. loss: 1.230 | lr: 0.0002970933397858569\n",
            "Epoch: 1 | Step: 13800 | Avg. loss: 1.214 | lr: 0.00029633718468332227\n",
            "Epoch: 1 | Step: 13850 | Avg. loss: 1.208 | lr: 0.00029558102958078764\n",
            "Epoch: 1 | Step: 13900 | Avg. loss: 1.225 | lr: 0.00029482487447825295\n",
            "Epoch: 1 | Step: 13950 | Avg. loss: 1.174 | lr: 0.0002940687193757183\n",
            "Epoch: 1 | Step: 14000 | Avg. loss: 1.222 | lr: 0.0002933125642731837\n",
            "Saving model with test loss of 1.348\n",
            "Epoch: 1 | Step: 14050 | Avg. loss: 1.166 | lr: 0.0002925564091706491\n",
            "Epoch: 1 | Step: 14100 | Avg. loss: 1.181 | lr: 0.00029180025406811447\n",
            "Epoch: 1 | Step: 14150 | Avg. loss: 1.268 | lr: 0.00029104409896557983\n",
            "Epoch: 1 | Step: 14200 | Avg. loss: 1.218 | lr: 0.0002902879438630452\n",
            "Epoch: 1 | Step: 14250 | Avg. loss: 1.243 | lr: 0.00028953178876051056\n",
            "Epoch: 1 | Step: 14300 | Avg. loss: 1.241 | lr: 0.00028877563365797593\n",
            "Epoch: 1 | Step: 14350 | Avg. loss: 1.217 | lr: 0.0002880194785554413\n",
            "Epoch: 1 | Step: 14400 | Avg. loss: 1.203 | lr: 0.00028726332345290666\n",
            "Epoch: 1 | Step: 14450 | Avg. loss: 1.194 | lr: 0.000286507168350372\n",
            "Epoch: 1 | Step: 14500 | Avg. loss: 1.212 | lr: 0.00028575101324783745\n",
            "Epoch: 1 | Step: 14550 | Avg. loss: 1.138 | lr: 0.0002849948581453028\n",
            "Epoch: 1 | Step: 14600 | Avg. loss: 1.192 | lr: 0.0002842387030427682\n",
            "Epoch: 1 | Step: 14650 | Avg. loss: 1.159 | lr: 0.0002834825479402335\n",
            "Epoch: 1 | Step: 14700 | Avg. loss: 1.161 | lr: 0.00028272639283769886\n",
            "Epoch: 1 | Step: 14750 | Avg. loss: 1.177 | lr: 0.0002819702377351642\n",
            "Epoch: 1 | Step: 14800 | Avg. loss: 1.178 | lr: 0.0002812140826326296\n",
            "Epoch: 1 | Step: 14850 | Avg. loss: 1.121 | lr: 0.00028045792753009495\n",
            "Epoch: 1 | Step: 14900 | Avg. loss: 1.196 | lr: 0.0002797017724275603\n",
            "Epoch: 1 | Step: 14950 | Avg. loss: 1.221 | lr: 0.00027894561732502574\n",
            "Epoch: 1 | Step: 15000 | Avg. loss: 1.205 | lr: 0.0002781894622224911\n",
            "Saving model with test loss of 1.300\n",
            "Epoch: 1 | Step: 15050 | Avg. loss: 1.194 | lr: 0.00027743330711995647\n",
            "Epoch: 1 | Step: 15100 | Avg. loss: 1.207 | lr: 0.00027667715201742184\n",
            "Epoch: 1 | Step: 15150 | Avg. loss: 1.178 | lr: 0.0002759209969148872\n",
            "Epoch: 1 | Step: 15200 | Avg. loss: 1.127 | lr: 0.00027516484181235257\n",
            "Epoch: 1 | Step: 15250 | Avg. loss: 1.167 | lr: 0.00027440868670981794\n",
            "Epoch: 1 | Step: 15300 | Avg. loss: 1.138 | lr: 0.0002736525316072833\n",
            "Epoch: 1 | Step: 15350 | Avg. loss: 1.169 | lr: 0.0002728963765047486\n",
            "Epoch: 1 | Step: 15400 | Avg. loss: 1.203 | lr: 0.000272140221402214\n",
            "Epoch: 1 | Step: 15450 | Avg. loss: 1.118 | lr: 0.0002713840662996794\n",
            "Epoch: 1 | Step: 15500 | Avg. loss: 1.173 | lr: 0.00027062791119714476\n",
            "Epoch: 1 | Step: 15550 | Avg. loss: 1.100 | lr: 0.00026987175609461013\n",
            "Epoch: 1 | Step: 15600 | Avg. loss: 1.123 | lr: 0.0002691156009920755\n",
            "Epoch: 1 | Step: 15650 | Avg. loss: 1.146 | lr: 0.00026835944588954086\n",
            "Epoch: 1 | Step: 15700 | Avg. loss: 1.125 | lr: 0.00026760329078700623\n",
            "Epoch: 1 | Step: 15750 | Avg. loss: 1.108 | lr: 0.0002668471356844716\n",
            "Epoch: 1 | Step: 15800 | Avg. loss: 1.138 | lr: 0.00026609098058193696\n",
            "Epoch: 1 | Step: 15850 | Avg. loss: 1.178 | lr: 0.0002653348254794023\n",
            "Epoch: 1 | Step: 15900 | Avg. loss: 1.172 | lr: 0.00026457867037686775\n",
            "Epoch: 1 | Step: 15950 | Avg. loss: 1.130 | lr: 0.0002638225152743331\n",
            "Epoch: 1 | Step: 16000 | Avg. loss: 1.143 | lr: 0.0002630663601717985\n",
            "Saving model with test loss of 1.522\n",
            "Epoch: 1 | Step: 16050 | Avg. loss: 1.163 | lr: 0.0002623102050692638\n",
            "Epoch: 1 | Step: 16100 | Avg. loss: 1.159 | lr: 0.00026155404996672916\n",
            "Epoch: 1 | Step: 16150 | Avg. loss: 1.170 | lr: 0.0002607978948641945\n",
            "Epoch: 1 | Step: 16200 | Avg. loss: 1.099 | lr: 0.0002600417397616599\n",
            "Epoch: 1 | Step: 16250 | Avg. loss: 1.104 | lr: 0.00025928558465912525\n",
            "Epoch: 1 | Step: 16300 | Avg. loss: 1.120 | lr: 0.0002585294295565906\n",
            "Epoch: 1 | Step: 16350 | Avg. loss: 1.165 | lr: 0.00025777327445405604\n",
            "Epoch: 1 | Step: 16400 | Avg. loss: 1.189 | lr: 0.0002570171193515214\n",
            "Epoch: 1 | Step: 16450 | Avg. loss: 1.187 | lr: 0.00025626096424898677\n",
            "Epoch: 1 | Step: 16500 | Avg. loss: 1.127 | lr: 0.00025550480914645214\n",
            "Epoch: 1 | Step: 16550 | Avg. loss: 1.155 | lr: 0.0002547486540439175\n",
            "Epoch: 1 | Step: 16600 | Avg. loss: 1.190 | lr: 0.00025399249894138287\n",
            "Epoch: 1 | Step: 16650 | Avg. loss: 1.160 | lr: 0.00025323634383884823\n",
            "Epoch: 1 | Step: 16700 | Avg. loss: 1.141 | lr: 0.0002524801887363136\n",
            "Epoch: 1 | Step: 16750 | Avg. loss: 1.094 | lr: 0.00025172403363377897\n",
            "Epoch: 1 | Step: 16800 | Avg. loss: 1.136 | lr: 0.00025096787853124433\n",
            "Epoch: 1 | Step: 16850 | Avg. loss: 1.139 | lr: 0.0002502117234287097\n",
            "Epoch: 1 | Step: 16900 | Avg. loss: 1.148 | lr: 0.00024945556832617506\n",
            "Epoch: 1 | Step: 16950 | Avg. loss: 1.146 | lr: 0.00024869941322364043\n",
            "Epoch: 1 | Step: 17000 | Avg. loss: 1.089 | lr: 0.0002479432581211058\n",
            "Saving model with test loss of 1.403\n",
            "Epoch: 1 | Step: 17050 | Avg. loss: 1.159 | lr: 0.00024718710301857116\n",
            "Epoch: 1 | Step: 17100 | Avg. loss: 1.171 | lr: 0.00024643094791603653\n",
            "Epoch: 1 | Step: 17150 | Avg. loss: 1.117 | lr: 0.0002456747928135019\n",
            "Epoch: 1 | Step: 17200 | Avg. loss: 1.133 | lr: 0.00024491863771096726\n",
            "Epoch: 1 | Step: 17250 | Avg. loss: 1.120 | lr: 0.0002441624826084327\n",
            "Epoch: 1 | Step: 17300 | Avg. loss: 1.148 | lr: 0.00024340632750589802\n",
            "Epoch: 1 | Step: 17350 | Avg. loss: 1.141 | lr: 0.00024265017240336338\n",
            "Epoch: 1 | Step: 17400 | Avg. loss: 1.139 | lr: 0.00024189401730082875\n",
            "Epoch: 1 | Step: 17450 | Avg. loss: 1.135 | lr: 0.00024113786219829412\n",
            "Epoch: 1 | Step: 17500 | Avg. loss: 1.095 | lr: 0.00024038170709575948\n",
            "Epoch: 1 | Step: 17550 | Avg. loss: 1.121 | lr: 0.00023962555199322485\n",
            "Epoch: 1 | Step: 17600 | Avg. loss: 1.082 | lr: 0.00023886939689069024\n",
            "Epoch: 1 | Step: 17650 | Avg. loss: 1.137 | lr: 0.0002381132417881556\n",
            "Epoch: 1 | Step: 17700 | Avg. loss: 1.094 | lr: 0.00023735708668562095\n",
            "Epoch: 1 | Step: 17750 | Avg. loss: 1.102 | lr: 0.0002366009315830863\n",
            "Epoch: 1 | Step: 17800 | Avg. loss: 1.128 | lr: 0.0002358447764805517\n",
            "Epoch: 1 | Step: 17850 | Avg. loss: 1.129 | lr: 0.00023508862137801707\n",
            "Epoch: 1 | Step: 17900 | Avg. loss: 1.134 | lr: 0.00023433246627548244\n",
            "Epoch: 1 | Step: 17950 | Avg. loss: 1.186 | lr: 0.0002335763111729478\n",
            "Epoch: 1 | Step: 18000 | Avg. loss: 1.141 | lr: 0.00023282015607041317\n",
            "Saving model with test loss of 1.418\n",
            "Epoch: 1 | Step: 18050 | Avg. loss: 1.113 | lr: 0.00023206400096787853\n",
            "Epoch: 1 | Step: 18100 | Avg. loss: 1.146 | lr: 0.0002313078458653439\n",
            "Epoch: 1 | Step: 18150 | Avg. loss: 1.157 | lr: 0.00023055169076280927\n",
            "Epoch: 1 | Step: 18200 | Avg. loss: 1.074 | lr: 0.00022979553566027463\n",
            "Epoch: 1 | Step: 18250 | Avg. loss: 1.183 | lr: 0.00022903938055774002\n",
            "Epoch: 1 | Step: 18300 | Avg. loss: 1.137 | lr: 0.0002282832254552054\n",
            "Epoch: 1 | Step: 18350 | Avg. loss: 1.083 | lr: 0.00022752707035267076\n",
            "Epoch: 1 | Step: 18400 | Avg. loss: 1.099 | lr: 0.0002267709152501361\n",
            "Epoch: 1 | Step: 18450 | Avg. loss: 1.149 | lr: 0.00022601476014760146\n",
            "Epoch: 1 | Step: 18500 | Avg. loss: 1.142 | lr: 0.00022525860504506685\n",
            "Epoch: 1 | Step: 18550 | Avg. loss: 1.187 | lr: 0.00022450244994253222\n",
            "Epoch: 1 | Step: 18600 | Avg. loss: 1.093 | lr: 0.00022374629483999759\n",
            "Epoch: 1 | Step: 18650 | Avg. loss: 1.067 | lr: 0.00022299013973746295\n",
            "Epoch: 1 | Step: 18700 | Avg. loss: 1.084 | lr: 0.00022223398463492835\n",
            "Epoch: 1 | Step: 18750 | Avg. loss: 1.089 | lr: 0.00022147782953239368\n",
            "Epoch: 1 | Step: 18800 | Avg. loss: 1.093 | lr: 0.00022072167442985905\n",
            "Epoch: 1 | Step: 18850 | Avg. loss: 1.061 | lr: 0.00021996551932732442\n",
            "Epoch: 1 | Step: 18900 | Avg. loss: 1.042 | lr: 0.00021920936422478978\n",
            "Epoch: 1 | Step: 18950 | Avg. loss: 1.064 | lr: 0.00021845320912225517\n",
            "Epoch: 1 | Step: 19000 | Avg. loss: 1.056 | lr: 0.00021769705401972054\n",
            "Saving model with test loss of 1.317\n",
            "Epoch: 1 | Step: 19050 | Avg. loss: 1.094 | lr: 0.0002169408989171859\n",
            "Epoch: 1 | Step: 19100 | Avg. loss: 1.080 | lr: 0.00021618474381465125\n",
            "Epoch: 1 | Step: 19150 | Avg. loss: 1.088 | lr: 0.0002154285887121166\n",
            "Epoch: 1 | Step: 19200 | Avg. loss: 1.086 | lr: 0.000214672433609582\n",
            "Epoch: 1 | Step: 19250 | Avg. loss: 1.089 | lr: 0.00021391627850704737\n",
            "Epoch: 1 | Step: 19300 | Avg. loss: 1.071 | lr: 0.00021316012340451274\n",
            "Epoch: 1 | Step: 19350 | Avg. loss: 1.095 | lr: 0.0002124039683019781\n",
            "Epoch: 1 | Step: 19400 | Avg. loss: 1.039 | lr: 0.0002116478131994435\n",
            "Epoch: 1 | Step: 19450 | Avg. loss: 1.131 | lr: 0.00021089165809690886\n",
            "Epoch: 1 | Step: 19500 | Avg. loss: 1.114 | lr: 0.0002101355029943742\n",
            "Epoch: 1 | Step: 19550 | Avg. loss: 1.158 | lr: 0.00020937934789183957\n",
            "Epoch: 1 | Step: 19600 | Avg. loss: 1.066 | lr: 0.00020862319278930493\n",
            "Epoch: 1 | Step: 19650 | Avg. loss: 1.146 | lr: 0.00020786703768677032\n",
            "Epoch: 1 | Step: 19700 | Avg. loss: 1.118 | lr: 0.0002071108825842357\n",
            "Epoch: 1 | Step: 19750 | Avg. loss: 1.123 | lr: 0.00020635472748170106\n",
            "Epoch: 1 | Step: 19800 | Avg. loss: 1.062 | lr: 0.00020559857237916642\n",
            "Epoch: 1 | Step: 19850 | Avg. loss: 1.114 | lr: 0.0002048424172766318\n",
            "Epoch: 1 | Step: 19900 | Avg. loss: 1.042 | lr: 0.00020408626217409715\n",
            "Epoch: 1 | Step: 19950 | Avg. loss: 1.005 | lr: 0.00020333010707156252\n",
            "Epoch: 1 | Step: 20000 | Avg. loss: 1.118 | lr: 0.00020257395196902789\n",
            "Saving model with test loss of 1.398\n",
            "Epoch: 1 | Step: 20050 | Avg. loss: 1.075 | lr: 0.00020181779686649325\n",
            "Epoch: 1 | Step: 20100 | Avg. loss: 1.053 | lr: 0.00020106164176395864\n",
            "Epoch: 1 | Step: 20150 | Avg. loss: 1.122 | lr: 0.000200305486661424\n",
            "Epoch: 1 | Step: 20200 | Avg. loss: 1.094 | lr: 0.00019954933155888935\n",
            "Epoch: 1 | Step: 20250 | Avg. loss: 1.097 | lr: 0.00019879317645635471\n",
            "Epoch: 1 | Step: 20300 | Avg. loss: 1.010 | lr: 0.0001980370213538201\n",
            "Epoch: 1 | Step: 20350 | Avg. loss: 1.072 | lr: 0.00019728086625128547\n",
            "Epoch: 1 | Step: 20400 | Avg. loss: 1.036 | lr: 0.00019652471114875084\n",
            "Epoch: 1 | Step: 20450 | Avg. loss: 1.063 | lr: 0.0001957685560462162\n",
            "Epoch: 1 | Step: 20500 | Avg. loss: 1.122 | lr: 0.00019501240094368157\n",
            "Epoch: 1 | Step: 20550 | Avg. loss: 1.081 | lr: 0.00019425624584114694\n",
            "Epoch: 1 | Step: 20600 | Avg. loss: 1.067 | lr: 0.0001935000907386123\n",
            "Epoch: 1 | Step: 20650 | Avg. loss: 1.000 | lr: 0.00019274393563607767\n",
            "Epoch: 1 | Step: 20700 | Avg. loss: 1.074 | lr: 0.00019198778053354304\n",
            "Epoch: 1 | Step: 20750 | Avg. loss: 1.102 | lr: 0.00019123162543100843\n",
            "Epoch: 1 | Step: 20800 | Avg. loss: 1.040 | lr: 0.0001904754703284738\n",
            "Epoch: 1 | Step: 20850 | Avg. loss: 1.083 | lr: 0.00018971931522593916\n",
            "Epoch: 1 | Step: 20900 | Avg. loss: 1.086 | lr: 0.0001889631601234045\n",
            "Epoch: 1 | Step: 20950 | Avg. loss: 1.045 | lr: 0.00018820700502086986\n",
            "Epoch: 1 | Step: 21000 | Avg. loss: 1.022 | lr: 0.00018745084991833526\n",
            "Saving model with test loss of 1.393\n",
            "Epoch: 1 | Step: 21050 | Avg. loss: 1.135 | lr: 0.00018669469481580062\n",
            "Epoch: 1 | Step: 21100 | Avg. loss: 1.087 | lr: 0.000185938539713266\n",
            "Epoch: 1 | Step: 21150 | Avg. loss: 1.058 | lr: 0.00018518238461073136\n",
            "Epoch: 1 | Step: 21200 | Avg. loss: 1.069 | lr: 0.00018442622950819675\n",
            "Epoch: 1 | Step: 21250 | Avg. loss: 1.058 | lr: 0.0001836700744056621\n",
            "Epoch: 1 | Step: 21300 | Avg. loss: 1.062 | lr: 0.00018291391930312745\n",
            "Epoch: 1 | Step: 21350 | Avg. loss: 1.035 | lr: 0.00018215776420059282\n",
            "Epoch: 1 | Step: 21400 | Avg. loss: 1.084 | lr: 0.00018140160909805818\n",
            "Epoch: 1 | Step: 21450 | Avg. loss: 1.070 | lr: 0.00018064545399552358\n",
            "Epoch: 1 | Step: 21500 | Avg. loss: 1.005 | lr: 0.00017988929889298894\n",
            "Epoch: 1 | Step: 21550 | Avg. loss: 1.042 | lr: 0.0001791331437904543\n",
            "Epoch: 1 | Step: 21600 | Avg. loss: 1.064 | lr: 0.00017837698868791968\n",
            "Epoch: 1 | Step: 21650 | Avg. loss: 1.029 | lr: 0.00017762083358538501\n",
            "Epoch: 1 | Step: 21700 | Avg. loss: 1.061 | lr: 0.0001768646784828504\n",
            "Epoch: 1 | Step: 21750 | Avg. loss: 1.096 | lr: 0.00017610852338031577\n",
            "Epoch: 1 | Step: 21800 | Avg. loss: 1.013 | lr: 0.00017535236827778114\n",
            "Epoch: 1 | Step: 21850 | Avg. loss: 1.075 | lr: 0.0001745962131752465\n",
            "Epoch: 1 | Step: 21900 | Avg. loss: 1.034 | lr: 0.0001738400580727119\n",
            "Epoch: 1 | Step: 21950 | Avg. loss: 1.024 | lr: 0.00017308390297017726\n",
            "Epoch: 1 | Step: 22000 | Avg. loss: 0.977 | lr: 0.0001723277478676426\n",
            "Saving model with test loss of 1.246\n",
            "Epoch: 1 | Step: 22050 | Avg. loss: 1.013 | lr: 0.00017157159276510797\n",
            "Epoch: 1 | Step: 22100 | Avg. loss: 1.016 | lr: 0.00017081543766257333\n",
            "Epoch: 1 | Step: 22150 | Avg. loss: 1.021 | lr: 0.00017005928256003873\n",
            "Epoch: 1 | Step: 22200 | Avg. loss: 1.024 | lr: 0.0001693031274575041\n",
            "Epoch: 1 | Step: 22250 | Avg. loss: 1.022 | lr: 0.00016854697235496946\n",
            "Epoch: 1 | Step: 22300 | Avg. loss: 1.017 | lr: 0.00016779081725243483\n",
            "Epoch: 1 | Step: 22350 | Avg. loss: 1.094 | lr: 0.0001670346621499002\n",
            "Epoch: 1 | Step: 22400 | Avg. loss: 1.015 | lr: 0.00016627850704736556\n",
            "Epoch: 1 | Step: 22450 | Avg. loss: 1.022 | lr: 0.00016552235194483092\n",
            "Epoch: 1 | Step: 22500 | Avg. loss: 0.994 | lr: 0.0001647661968422963\n",
            "Epoch: 1 | Step: 22550 | Avg. loss: 0.996 | lr: 0.00016401004173976165\n",
            "Epoch: 1 | Step: 22600 | Avg. loss: 1.038 | lr: 0.00016325388663722705\n",
            "Epoch: 1 | Step: 22650 | Avg. loss: 0.980 | lr: 0.0001624977315346924\n",
            "Epoch: 1 | Step: 22700 | Avg. loss: 1.011 | lr: 0.00016174157643215775\n",
            "Epoch: 1 | Step: 22750 | Avg. loss: 1.046 | lr: 0.00016098542132962312\n",
            "Epoch: 1 | Step: 22800 | Avg. loss: 1.022 | lr: 0.0001602292662270885\n",
            "Epoch: 1 | Step: 22850 | Avg. loss: 0.983 | lr: 0.00015947311112455388\n",
            "Epoch: 1 | Step: 22900 | Avg. loss: 1.023 | lr: 0.00015871695602201924\n",
            "Epoch: 1 | Step: 22950 | Avg. loss: 1.061 | lr: 0.0001579608009194846\n",
            "Epoch: 1 | Step: 23000 | Avg. loss: 1.048 | lr: 0.00015720464581694997\n",
            "Saving model with test loss of 1.257\n",
            "Epoch: 1 | Step: 23050 | Avg. loss: 1.014 | lr: 0.00015644849071441534\n",
            "Epoch: 1 | Step: 23100 | Avg. loss: 1.029 | lr: 0.0001556923356118807\n",
            "Epoch: 1 | Step: 23150 | Avg. loss: 1.048 | lr: 0.00015493618050934607\n",
            "Epoch: 1 | Step: 23200 | Avg. loss: 1.060 | lr: 0.00015418002540681144\n",
            "Epoch: 1 | Step: 23250 | Avg. loss: 0.993 | lr: 0.00015342387030427683\n",
            "Epoch: 1 | Step: 23300 | Avg. loss: 1.024 | lr: 0.0001526677152017422\n",
            "Epoch: 1 | Step: 23350 | Avg. loss: 1.041 | lr: 0.00015191156009920756\n",
            "Epoch: 1 | Step: 23400 | Avg. loss: 0.968 | lr: 0.0001511554049966729\n",
            "Epoch: 1 | Step: 23450 | Avg. loss: 1.023 | lr: 0.00015039924989413827\n",
            "Epoch: 1 | Step: 23500 | Avg. loss: 0.983 | lr: 0.00014964309479160366\n",
            "Epoch: 1 | Step: 23550 | Avg. loss: 1.013 | lr: 0.00014888693968906903\n",
            "Epoch: 1 | Step: 23600 | Avg. loss: 0.971 | lr: 0.0001481307845865344\n",
            "Epoch: 1 | Step: 23650 | Avg. loss: 1.025 | lr: 0.00014737462948399976\n",
            "Epoch: 1 | Step: 23700 | Avg. loss: 1.024 | lr: 0.00014661847438146515\n",
            "Epoch: 1 | Step: 23750 | Avg. loss: 1.032 | lr: 0.00014586231927893052\n",
            "Epoch: 1 | Step: 23800 | Avg. loss: 1.022 | lr: 0.00014510616417639586\n",
            "Epoch: 1 | Step: 23850 | Avg. loss: 1.023 | lr: 0.00014435000907386122\n",
            "Epoch: 1 | Step: 23900 | Avg. loss: 0.989 | lr: 0.0001435938539713266\n",
            "Epoch: 1 | Step: 23950 | Avg. loss: 0.977 | lr: 0.00014283769886879198\n",
            "Epoch: 1 | Step: 24000 | Avg. loss: 0.994 | lr: 0.00014208154376625735\n",
            "Saving model with test loss of 1.216\n",
            "Epoch: 1 | Step: 24050 | Avg. loss: 1.047 | lr: 0.0001413253886637227\n",
            "Epoch: 1 | Step: 24100 | Avg. loss: 1.035 | lr: 0.00014056923356118808\n",
            "Epoch: 1 | Step: 24150 | Avg. loss: 0.967 | lr: 0.00013981307845865342\n",
            "Epoch: 1 | Step: 24200 | Avg. loss: 0.985 | lr: 0.0001390569233561188\n",
            "Epoch: 1 | Step: 24250 | Avg. loss: 1.027 | lr: 0.00013830076825358418\n",
            "Epoch: 1 | Step: 24300 | Avg. loss: 1.038 | lr: 0.00013754461315104954\n",
            "Epoch: 1 | Step: 24350 | Avg. loss: 1.034 | lr: 0.0001367884580485149\n",
            "Epoch: 1 | Step: 24400 | Avg. loss: 0.974 | lr: 0.0001360323029459803\n",
            "Epoch: 1 | Step: 24450 | Avg. loss: 1.015 | lr: 0.00013527614784344567\n",
            "Epoch: 1 | Step: 24500 | Avg. loss: 1.036 | lr: 0.000134519992740911\n",
            "Epoch: 1 | Step: 24550 | Avg. loss: 0.996 | lr: 0.00013376383763837637\n",
            "Epoch: 1 | Step: 24600 | Avg. loss: 1.022 | lr: 0.00013300768253584174\n",
            "Epoch: 1 | Step: 24650 | Avg. loss: 1.062 | lr: 0.00013225152743330713\n",
            "Epoch: 1 | Step: 24700 | Avg. loss: 0.992 | lr: 0.0001314953723307725\n",
            "Epoch: 1 | Step: 24750 | Avg. loss: 1.003 | lr: 0.00013073921722823786\n",
            "Epoch: 1 | Step: 24800 | Avg. loss: 0.934 | lr: 0.00012998306212570323\n",
            "Epoch: 1 | Step: 24850 | Avg. loss: 1.039 | lr: 0.0001292269070231686\n",
            "Epoch: 1 | Step: 24900 | Avg. loss: 1.036 | lr: 0.00012847075192063396\n",
            "Epoch: 1 | Step: 24950 | Avg. loss: 0.970 | lr: 0.00012771459681809933\n",
            "Epoch: 1 | Step: 25000 | Avg. loss: 0.984 | lr: 0.0001269584417155647\n",
            "Saving model with test loss of 1.332\n",
            "Epoch: 1 | Step: 25050 | Avg. loss: 0.971 | lr: 0.00012620228661303006\n",
            "Epoch: 1 | Step: 25100 | Avg. loss: 1.026 | lr: 0.00012544613151049545\n",
            "Epoch: 1 | Step: 25150 | Avg. loss: 0.977 | lr: 0.0001246899764079608\n",
            "Epoch: 1 | Step: 25200 | Avg. loss: 0.978 | lr: 0.00012393382130542618\n",
            "Epoch: 1 | Step: 25250 | Avg. loss: 1.044 | lr: 0.00012317766620289155\n",
            "Epoch: 1 | Step: 25300 | Avg. loss: 1.000 | lr: 0.00012242151110035691\n",
            "Epoch: 1 | Step: 25350 | Avg. loss: 0.973 | lr: 0.00012166535599782228\n",
            "Epoch: 1 | Step: 25400 | Avg. loss: 0.998 | lr: 0.00012090920089528765\n",
            "Epoch: 1 | Step: 25450 | Avg. loss: 1.012 | lr: 0.00012015304579275301\n",
            "Epoch: 1 | Step: 25500 | Avg. loss: 0.973 | lr: 0.00011939689069021838\n",
            "Epoch: 1 | Step: 25550 | Avg. loss: 0.984 | lr: 0.00011864073558768374\n",
            "Epoch: 1 | Step: 25600 | Avg. loss: 1.016 | lr: 0.00011788458048514912\n",
            "Epoch: 1 | Step: 25650 | Avg. loss: 0.932 | lr: 0.00011712842538261448\n",
            "Epoch: 1 | Step: 25700 | Avg. loss: 0.986 | lr: 0.00011637227028007986\n",
            "Epoch: 1 | Step: 25750 | Avg. loss: 0.963 | lr: 0.00011561611517754522\n",
            "Epoch: 1 | Step: 25800 | Avg. loss: 1.029 | lr: 0.0001148599600750106\n",
            "Epoch: 1 | Step: 25850 | Avg. loss: 1.013 | lr: 0.00011410380497247595\n",
            "Epoch: 1 | Step: 25900 | Avg. loss: 1.012 | lr: 0.00011334764986994132\n",
            "Epoch: 1 | Step: 25950 | Avg. loss: 0.981 | lr: 0.0001125914947674067\n",
            "Epoch: 1 | Step: 26000 | Avg. loss: 0.976 | lr: 0.00011183533966487205\n",
            "Saving model with test loss of 1.069\n",
            "Epoch: 1 | Step: 26050 | Avg. loss: 0.989 | lr: 0.00011107918456233743\n",
            "Epoch: 1 | Step: 26100 | Avg. loss: 0.974 | lr: 0.0001103230294598028\n",
            "Epoch: 1 | Step: 26150 | Avg. loss: 0.992 | lr: 0.00010956687435726818\n",
            "Epoch: 1 | Step: 26200 | Avg. loss: 0.953 | lr: 0.00010881071925473353\n",
            "Epoch: 1 | Step: 26250 | Avg. loss: 1.030 | lr: 0.00010805456415219891\n",
            "Epoch: 1 | Step: 26300 | Avg. loss: 0.971 | lr: 0.00010729840904966427\n",
            "Epoch: 1 | Step: 26350 | Avg. loss: 0.973 | lr: 0.00010654225394712964\n",
            "Epoch: 1 | Step: 26400 | Avg. loss: 0.972 | lr: 0.000105786098844595\n",
            "Epoch: 1 | Step: 26450 | Avg. loss: 0.952 | lr: 0.00010502994374206037\n",
            "Epoch: 1 | Step: 26500 | Avg. loss: 0.977 | lr: 0.00010427378863952575\n",
            "Epoch: 1 | Step: 26550 | Avg. loss: 1.000 | lr: 0.0001035176335369911\n",
            "Epoch: 1 | Step: 26600 | Avg. loss: 1.016 | lr: 0.00010276147843445648\n",
            "Epoch: 1 | Step: 26650 | Avg. loss: 0.988 | lr: 0.00010200532333192185\n",
            "Epoch: 1 | Step: 26700 | Avg. loss: 1.012 | lr: 0.00010124916822938721\n",
            "Epoch: 1 | Step: 26750 | Avg. loss: 0.962 | lr: 0.00010049301312685258\n",
            "Epoch: 1 | Step: 26800 | Avg. loss: 0.946 | lr: 9.973685802431795e-05\n",
            "Epoch: 1 | Step: 26850 | Avg. loss: 0.952 | lr: 9.898070292178332e-05\n",
            "Epoch: 1 | Step: 26900 | Avg. loss: 0.991 | lr: 9.822454781924868e-05\n",
            "Epoch: 1 | Step: 26950 | Avg. loss: 0.990 | lr: 9.746839271671406e-05\n",
            "Epoch: 1 | Step: 27000 | Avg. loss: 0.968 | lr: 9.671223761417942e-05\n",
            "Saving model with test loss of 1.528\n",
            "Epoch: 1 | Step: 27050 | Avg. loss: 0.950 | lr: 9.59560825116448e-05\n",
            "Epoch: 1 | Step: 27100 | Avg. loss: 0.966 | lr: 9.519992740911015e-05\n",
            "Epoch: 1 | Step: 27150 | Avg. loss: 0.966 | lr: 9.444377230657552e-05\n",
            "Epoch: 1 | Step: 27200 | Avg. loss: 0.905 | lr: 9.36876172040409e-05\n",
            "Epoch: 1 | Step: 27250 | Avg. loss: 0.986 | lr: 9.293146210150625e-05\n",
            "Epoch: 1 | Step: 27300 | Avg. loss: 0.939 | lr: 9.217530699897163e-05\n",
            "Epoch: 1 | Step: 27350 | Avg. loss: 0.931 | lr: 9.1419151896437e-05\n",
            "Epoch: 1 | Step: 27400 | Avg. loss: 0.978 | lr: 9.066299679390238e-05\n",
            "Epoch: 1 | Step: 27450 | Avg. loss: 0.897 | lr: 8.990684169136773e-05\n",
            "Epoch: 1 | Step: 27500 | Avg. loss: 0.924 | lr: 8.915068658883311e-05\n",
            "Epoch: 1 | Step: 27550 | Avg. loss: 1.023 | lr: 8.839453148629847e-05\n",
            "Epoch: 1 | Step: 27600 | Avg. loss: 0.938 | lr: 8.763837638376384e-05\n",
            "Epoch: 1 | Step: 27650 | Avg. loss: 0.981 | lr: 8.68822212812292e-05\n",
            "Epoch: 1 | Step: 27700 | Avg. loss: 0.985 | lr: 8.612606617869457e-05\n",
            "Epoch: 1 | Step: 27750 | Avg. loss: 0.972 | lr: 8.536991107615995e-05\n",
            "Epoch: 1 | Step: 27800 | Avg. loss: 0.914 | lr: 8.46137559736253e-05\n",
            "Epoch: 1 | Step: 27850 | Avg. loss: 0.900 | lr: 8.385760087109068e-05\n",
            "Epoch: 1 | Step: 27900 | Avg. loss: 0.977 | lr: 8.310144576855605e-05\n",
            "Epoch: 1 | Step: 27950 | Avg. loss: 0.922 | lr: 8.234529066602142e-05\n",
            "Epoch: 1 | Step: 28000 | Avg. loss: 0.936 | lr: 8.158913556348678e-05\n",
            "Saving model with test loss of 1.382\n",
            "Epoch: 1 | Step: 28050 | Avg. loss: 0.899 | lr: 8.083298046095215e-05\n",
            "Epoch: 1 | Step: 28100 | Avg. loss: 0.958 | lr: 8.007682535841753e-05\n",
            "Epoch: 1 | Step: 28150 | Avg. loss: 0.979 | lr: 7.932067025588288e-05\n",
            "Epoch: 1 | Step: 28200 | Avg. loss: 0.984 | lr: 7.856451515334826e-05\n",
            "Epoch: 1 | Step: 28250 | Avg. loss: 0.969 | lr: 7.780836005081362e-05\n",
            "Epoch: 1 | Step: 28300 | Avg. loss: 0.918 | lr: 7.7052204948279e-05\n",
            "Epoch: 1 | Step: 28350 | Avg. loss: 0.926 | lr: 7.629604984574436e-05\n",
            "Epoch: 1 | Step: 28400 | Avg. loss: 0.933 | lr: 7.553989474320972e-05\n",
            "Epoch: 1 | Step: 28450 | Avg. loss: 0.925 | lr: 7.47837396406751e-05\n",
            "Epoch: 1 | Step: 28500 | Avg. loss: 0.892 | lr: 7.402758453814047e-05\n",
            "Epoch: 1 | Step: 28550 | Avg. loss: 0.963 | lr: 7.327142943560583e-05\n",
            "Epoch: 1 | Step: 28600 | Avg. loss: 0.893 | lr: 7.25152743330712e-05\n",
            "Epoch: 1 | Step: 28650 | Avg. loss: 1.014 | lr: 7.175911923053658e-05\n",
            "Epoch: 1 | Step: 28700 | Avg. loss: 0.883 | lr: 7.100296412800193e-05\n",
            "Epoch: 1 | Step: 28750 | Avg. loss: 0.936 | lr: 7.024680902546731e-05\n",
            "Epoch: 1 | Step: 28800 | Avg. loss: 0.976 | lr: 6.949065392293268e-05\n",
            "Epoch: 1 | Step: 28850 | Avg. loss: 0.957 | lr: 6.873449882039804e-05\n",
            "Epoch: 1 | Step: 28900 | Avg. loss: 0.975 | lr: 6.797834371786341e-05\n",
            "Epoch: 1 | Step: 28950 | Avg. loss: 0.991 | lr: 6.722218861532877e-05\n",
            "Epoch: 1 | Step: 29000 | Avg. loss: 0.955 | lr: 6.646603351279415e-05\n",
            "Saving model with test loss of 1.193\n",
            "Epoch: 1 | Step: 29050 | Avg. loss: 0.936 | lr: 6.57098784102595e-05\n",
            "Epoch: 1 | Step: 29100 | Avg. loss: 0.940 | lr: 6.495372330772489e-05\n",
            "Epoch: 1 | Step: 29150 | Avg. loss: 0.892 | lr: 6.419756820519025e-05\n",
            "Epoch: 1 | Step: 29200 | Avg. loss: 0.952 | lr: 6.344141310265562e-05\n",
            "Epoch: 1 | Step: 29250 | Avg. loss: 0.936 | lr: 6.268525800012098e-05\n",
            "Epoch: 1 | Step: 29300 | Avg. loss: 0.971 | lr: 6.192910289758635e-05\n",
            "Epoch: 1 | Step: 29350 | Avg. loss: 0.958 | lr: 6.117294779505173e-05\n",
            "Epoch: 1 | Step: 29400 | Avg. loss: 1.005 | lr: 6.041679269251709e-05\n",
            "Epoch: 1 | Step: 29450 | Avg. loss: 0.901 | lr: 5.966063758998245e-05\n",
            "Epoch: 1 | Step: 29500 | Avg. loss: 0.937 | lr: 5.8904482487447826e-05\n",
            "Epoch: 1 | Step: 29550 | Avg. loss: 0.865 | lr: 5.814832738491319e-05\n",
            "Epoch: 1 | Step: 29600 | Avg. loss: 0.978 | lr: 5.7392172282378564e-05\n",
            "Epoch: 1 | Step: 29650 | Avg. loss: 0.899 | lr: 5.663601717984393e-05\n",
            "Epoch: 1 | Step: 29700 | Avg. loss: 0.921 | lr: 5.58798620773093e-05\n",
            "Epoch: 1 | Step: 29750 | Avg. loss: 0.958 | lr: 5.512370697477467e-05\n",
            "Epoch: 1 | Step: 29800 | Avg. loss: 0.882 | lr: 5.436755187224004e-05\n",
            "Epoch: 1 | Step: 29850 | Avg. loss: 0.925 | lr: 5.36113967697054e-05\n",
            "Epoch: 1 | Step: 29900 | Avg. loss: 0.951 | lr: 5.285524166717077e-05\n",
            "Epoch: 1 | Step: 29950 | Avg. loss: 0.963 | lr: 5.209908656463614e-05\n",
            "Epoch: 1 | Step: 30000 | Avg. loss: 0.884 | lr: 5.1342931462101505e-05\n",
            "Saving model with test loss of 1.135\n",
            "Epoch: 1 | Step: 30050 | Avg. loss: 0.936 | lr: 5.058677635956688e-05\n",
            "Epoch: 1 | Step: 30100 | Avg. loss: 0.915 | lr: 4.9830621257032244e-05\n",
            "Epoch: 1 | Step: 30150 | Avg. loss: 0.951 | lr: 4.9074466154497616e-05\n",
            "Epoch: 1 | Step: 30200 | Avg. loss: 0.930 | lr: 4.831831105196298e-05\n",
            "Epoch: 1 | Step: 30250 | Avg. loss: 0.925 | lr: 4.756215594942834e-05\n",
            "Epoch: 1 | Step: 30300 | Avg. loss: 0.920 | lr: 4.6806000846893714e-05\n",
            "Epoch: 1 | Step: 30350 | Avg. loss: 0.964 | lr: 4.604984574435908e-05\n",
            "Epoch: 1 | Step: 30400 | Avg. loss: 0.929 | lr: 4.529369064182445e-05\n",
            "Epoch: 1 | Step: 30450 | Avg. loss: 0.937 | lr: 4.453753553928982e-05\n",
            "Epoch: 1 | Step: 30500 | Avg. loss: 0.933 | lr: 4.378138043675519e-05\n",
            "Epoch: 1 | Step: 30550 | Avg. loss: 0.927 | lr: 4.302522533422056e-05\n",
            "Epoch: 1 | Step: 30600 | Avg. loss: 0.908 | lr: 4.226907023168593e-05\n",
            "Epoch: 1 | Step: 30650 | Avg. loss: 0.946 | lr: 4.151291512915129e-05\n",
            "Epoch: 1 | Step: 30700 | Avg. loss: 0.920 | lr: 4.0756760026616655e-05\n",
            "Epoch: 1 | Step: 30750 | Avg. loss: 0.915 | lr: 4.000060492408203e-05\n",
            "Epoch: 1 | Step: 30800 | Avg. loss: 0.923 | lr: 3.9244449821547393e-05\n",
            "Epoch: 1 | Step: 30850 | Avg. loss: 0.916 | lr: 3.8488294719012766e-05\n",
            "Epoch: 1 | Step: 30900 | Avg. loss: 0.894 | lr: 3.773213961647813e-05\n",
            "Epoch: 1 | Step: 30950 | Avg. loss: 0.858 | lr: 3.6975984513943505e-05\n",
            "Epoch: 1 | Step: 31000 | Avg. loss: 0.954 | lr: 3.621982941140887e-05\n",
            "Saving model with test loss of 1.326\n",
            "Epoch: 1 | Step: 31050 | Avg. loss: 0.901 | lr: 3.546367430887424e-05\n",
            "Epoch: 1 | Step: 31100 | Avg. loss: 0.916 | lr: 3.47075192063396e-05\n",
            "Epoch: 1 | Step: 31150 | Avg. loss: 0.920 | lr: 3.395136410380497e-05\n",
            "Epoch: 1 | Step: 31200 | Avg. loss: 0.890 | lr: 3.319520900127034e-05\n",
            "Epoch: 1 | Step: 31250 | Avg. loss: 0.907 | lr: 3.243905389873571e-05\n",
            "Epoch: 1 | Step: 31300 | Avg. loss: 0.893 | lr: 3.168289879620108e-05\n",
            "Epoch: 1 | Step: 31350 | Avg. loss: 0.971 | lr: 3.0926743693666445e-05\n",
            "Epoch: 1 | Step: 31400 | Avg. loss: 0.991 | lr: 3.017058859113181e-05\n",
            "Epoch: 1 | Step: 31450 | Avg. loss: 0.934 | lr: 2.941443348859718e-05\n",
            "Epoch: 1 | Step: 31500 | Avg. loss: 0.910 | lr: 2.865827838606255e-05\n",
            "Epoch: 1 | Step: 31550 | Avg. loss: 0.897 | lr: 2.790212328352792e-05\n",
            "Epoch: 1 | Step: 31600 | Avg. loss: 0.928 | lr: 2.7145968180993285e-05\n",
            "Epoch: 1 | Step: 31650 | Avg. loss: 0.899 | lr: 2.6389813078458654e-05\n",
            "Epoch: 1 | Step: 31700 | Avg. loss: 0.928 | lr: 2.5633657975924024e-05\n",
            "Epoch: 1 | Step: 31750 | Avg. loss: 0.887 | lr: 2.4877502873389393e-05\n",
            "Epoch: 1 | Step: 31800 | Avg. loss: 0.905 | lr: 2.4121347770854755e-05\n",
            "Epoch: 1 | Step: 31850 | Avg. loss: 0.939 | lr: 2.3365192668320125e-05\n",
            "Epoch: 1 | Step: 31900 | Avg. loss: 0.901 | lr: 2.2609037565785494e-05\n",
            "Epoch: 1 | Step: 31950 | Avg. loss: 0.884 | lr: 2.1852882463250863e-05\n",
            "Epoch: 1 | Step: 32000 | Avg. loss: 0.962 | lr: 2.109672736071623e-05\n",
            "Saving model with test loss of 1.223\n",
            "Epoch: 1 | Step: 32050 | Avg. loss: 0.897 | lr: 2.03405722581816e-05\n",
            "Epoch: 1 | Step: 32100 | Avg. loss: 0.894 | lr: 1.9584417155646968e-05\n",
            "Epoch: 1 | Step: 32150 | Avg. loss: 0.925 | lr: 1.8828262053112337e-05\n",
            "Epoch: 1 | Step: 32200 | Avg. loss: 0.879 | lr: 1.8072106950577703e-05\n",
            "Epoch: 1 | Step: 32250 | Avg. loss: 0.901 | lr: 1.731595184804307e-05\n",
            "Epoch: 1 | Step: 32300 | Avg. loss: 0.924 | lr: 1.6559796745508438e-05\n",
            "Epoch: 1 | Step: 32350 | Avg. loss: 0.890 | lr: 1.5803641642973807e-05\n",
            "Epoch: 1 | Step: 32400 | Avg. loss: 0.878 | lr: 1.5047486540439175e-05\n",
            "Epoch: 1 | Step: 32450 | Avg. loss: 0.927 | lr: 1.4291331437904544e-05\n",
            "Epoch: 1 | Step: 32500 | Avg. loss: 0.946 | lr: 1.3535176335369912e-05\n",
            "Epoch: 1 | Step: 32550 | Avg. loss: 0.914 | lr: 1.277902123283528e-05\n",
            "Epoch: 1 | Step: 32600 | Avg. loss: 0.938 | lr: 1.2022866130300647e-05\n",
            "Epoch: 1 | Step: 32650 | Avg. loss: 0.888 | lr: 1.1266711027766016e-05\n",
            "Epoch: 1 | Step: 32700 | Avg. loss: 0.884 | lr: 1.0510555925231384e-05\n",
            "Epoch: 1 | Step: 32750 | Avg. loss: 0.930 | lr: 9.754400822696751e-06\n",
            "Epoch: 1 | Step: 32800 | Avg. loss: 0.902 | lr: 8.998245720162119e-06\n",
            "Epoch: 1 | Step: 32850 | Avg. loss: 0.925 | lr: 8.242090617627488e-06\n",
            "Epoch: 1 | Step: 32900 | Avg. loss: 0.887 | lr: 7.485935515092856e-06\n",
            "Epoch: 1 | Step: 32950 | Avg. loss: 0.909 | lr: 6.729780412558224e-06\n",
            "Epoch: 1 | Step: 33000 | Avg. loss: 0.872 | lr: 5.973625310023592e-06\n",
            "Saving model with test loss of 1.107\n",
            "Epoch: 1 | Step: 33050 | Avg. loss: 0.869 | lr: 5.21747020748896e-06\n",
            "Epoch: 1 | Step: 33100 | Avg. loss: 0.871 | lr: 4.461315104954329e-06\n",
            "Epoch: 1 | Step: 33150 | Avg. loss: 0.909 | lr: 3.7051600024196964e-06\n",
            "Epoch: 1 | Step: 33200 | Avg. loss: 0.925 | lr: 2.9490048998850644e-06\n",
            "Epoch: 1 | Step: 33250 | Avg. loss: 0.927 | lr: 2.1928497973504325e-06\n",
            "Epoch: 1 | Step: 33300 | Avg. loss: 0.903 | lr: 1.4366946948158007e-06\n",
            "Epoch: 1 | Step: 33350 | Avg. loss: 0.890 | lr: 6.805395922811688e-07\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "xj4fK6xYe5Uc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "window_size = 50\n",
        "smoothed_losses = []\n",
        "for i in range(len(losses)-window_size):\n",
        "  smoothed_losses.append(np.mean(losses[i:i+window_size]))\n",
        "\n",
        "plt.plot(smoothed_losses[100:])"
      ],
      "metadata": {
        "id": "W-N3T7ABtXCk",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 451
        },
        "outputId": "41c630bd-b1bf-4fa5-9396-c9e65a4a0b46"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7fa0b8515d20>]"
            ]
          },
          "metadata": {},
          "execution_count": 24
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiQAAAGhCAYAAABRZq+GAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6Y0lEQVR4nO3deXxU1f3/8fdMNhLCQEBW2UFCwhYQQQqExa0oQpcfihZRqAtWcatfpVVQW1u11aq1gCziUmtbQattWaqyiLKpgCi4Qtj3JTAJ2WfO749kJrkkQAIzOSTzej4ePkLu3Nx75sMl8/bcc851GWOMAAAALHLbbgAAAACBBAAAWEcgAQAA1hFIAACAdQQSAABgHYEEAABYRyABAADWEUgAAIB10bYbUFnGGPn94VnDze12he3YNRH1cKIepaiFE/Vwoh5O1KO4Bi6Xq1L71phA4vcbHTlyPOTHjY52KymprrzeHBUV+UN+/JqGejhRj1LUwol6OFEPJ+pRrGHDuoqKqlwg4ZYNAACwjkACAACsI5AAAADrCCQAAMA6AgkAALCOQAIAAKwjkAAAAOsIJAAAwDoCCQAAsI5AAgAArCOQAAAA6wgkAADAOgIJAACwLuIDiTFGxkT246EBALAt2nYDbDLG6PFX1youLlr/NzrNdnMAAIhYER1I8gp8+n7XMUlSYZFfbpfLcosAAIhMEX3LpmwA4aYNAAD2VDmQbN++XVOmTNHIkSOVmpqq4cOHV7jf3LlzdcUVV6hbt24aMWKEli5detaNDbkyHSKMIwEAwJ4qB5Lvv/9eH374odq0aaMOHTpUuM/8+fM1efJkDRs2TLNmzVJaWpruvPNOff7552fb3pByOwKJvXYAABDpqjyGZOjQobr00kslSZMmTdLGjRvL7fPnP/9ZV111le655x5J0sUXX6zvvvtOU6dO1axZs86uxSFV5pYNgQQAAGuq3EPidp/6R3bu3Klt27Zp2LBhju1XXnmlVq1apYKCgqqeMmzKjmE1jCIBAMCakM+yycjIkCS1a9fOsb1Dhw4qLCzUzp07T3qr53Sio0M7BtdV5nButzvkx6+JoqLcjq+RjnqUohZO1MOJejhRj6oLeSA5dqx4Gq3H43FsD3wfeL2q3G6XkpLqnl3jTuD3l/aKJCbWkadubEiPX5N5PPG2m3BOoR6lqIUT9XCiHk7Uo/JqzDokfr+R15sT0mOWnVnj9ebKV1AY0uPXRFFRbnk88cX18PltN8c66lGKWjhRDyfq4UQ9ink88ZXuJQp5IKlfv74kKSsrS40bNw5u93q9jtfPRFFR+P5SfT5/WI9f01APJ+pRilo4UQ8n6uFEPSov5De32rdvL6l0LElARkaGYmJi1KpVq1Cf8qwExrX6mWYDAIA1IQ8krVq1Utu2bbVo0SLH9gULFqhfv36KjT23xmm4AlNtyCMAAFhT5Vs2ubm5+vDDDyVJu3fvVnZ2djB89OnTRw0bNtTEiRN1//33q3Xr1urbt68WLFigL774Qq+//npoWx8CLpckQw8JAAA2VTmQHD58WHfffbdjW+D71157TX379tXw4cOVm5urWbNmaebMmWrXrp3+8pe/qGfPnqFpdQgFO0jIIwAAWFPlQNKyZUt9++23p91v1KhRGjVq1Bk1qjq5SrpIyCMAANgT8Su2BAa18nA9AADsIZAwqBUAAOsIJCV5hEGtAADYQyBxnX4fAAAQXgSSklEk9JAAAGAPgYQeEgAArIv4QBJIJH46SAAAsCbiA4m7dN6v1XYAABDJIj6QBJBHAACwJ+IDidvFoFYAAGyL+EDCoFYAAOwjkNBDAgCAdQSSwB/IIwAAWBPxgUTBpePtNgMAgEgW8YHEHRxEQiIBAMCWiA8kAQwhAQDAnogPJIEeEgIJAAD2RHwgCYwhMSQSAACsifhAEpj2SxwBAMAeAknJV3pIAACwh0ASvGVjtx0AAESyiA8kpYNaSSQAANgS8YFE9JAAAGBdxAcSN4NaAQCwLuIDSQC3bAAAsCfiAwmDWgEAsI9AErxlQyIBAMAWAgk9JAAAWEcgEc+yAQDANgJJoIeEWzYAAFhDIOGWDQAA1hFIWKkVAADrCCQlX8kjAADYQyChhwQAAOsIJMFBrQAAwBYCSclXOkgAALCHQMItGwAArCOQMO0XAADrCCTBZ9kAAABbCCQlX7llAwCAPQQSptkAAGAdgaQkj/jpIQEAwBoCiev0+wAAgPCK+EASGEVCDwkAAPZEfCBxB0e1Wm0GAAARLeIDSWBQq59AAgCANQSS4MJoJBIAAGwhkDDrFwAA6wgk4lk2AADYRiBh2i8AANYRSFxM+wUAwDYCCU/7BQDAuogPJG4XY0gAALAt4gNJAHkEAAB7Ij6QuII9JJYbAgBABIv4QOJmYTQAAKyL+EDC0vEAANhHIKGHBAAA6wgkzLIBAMC6sAWSxYsXa9SoUerZs6cGDBigu+++Wzt37gzX6c4Yz7IBAMC+sASSNWvW6M4771THjh01depU/frXv9Y333yj8ePHKy8vLxynPGPMsgEAwL7ocBx0/vz5atGihX7/+98HP/AbNmyoG2+8URs3blTv3r3DcdozwiwbAADsC0sPSVFRkerWrRsMI5JUr149SefuBz+zbAAAsCcsPSQ/+clP9O677+pvf/ubRowYoaNHj+pPf/qTUlNT1atXrzM+bnR06PNTlLv4mK4wHb+miYpyO75GOupRilo4UQ8n6uFEParOZcLUZbF06VL98pe/1PHjxyVJKSkpmj17ts4777wzOp4xxtHjEiqz3v1S/16eoVGXXKCxV6aG/PgAAOD0wtJDsm7dOj3wwAO65pprNHjwYB09elTTpk3TrbfeqjfeeEN16tSp8jH9fiOvNyfkbS0s8EmS8vIKlZl5POTHr2miotzyeOLl9ebK5/Pbbo511KMUtXCiHk7Uw4l6FPN44ivdSxSWQPL444/r4osv1qRJk4Lb0tLSNHjwYL377ru69tprz+i4RUXh+0v1+U1Yj1/T+Hx+6lEG9ShFLZyohxP1cKIelReWm1tbtmxR586dHduaNWumpKQk7dixIxynPGOs1AoAgH1hCSQtWrTQV1995di2e/duZWZm6vzzzw/HKc9YaSCx2w4AACJZWALJ6NGj9cEHH+jxxx/XypUrtWDBAk2YMEGNGjXSsGHDwnHKM+YWS8cDAGBbWMaQjB07VrGxsfr73/+ut956S3Xr1lVaWpqee+45JSUlheOUZ4weEgAA7AtLIHG5XLruuut03XXXhePwIRWYSuwnkQAAYE3Er9hCDwkAAPYRSAIP17PcDgAAIhmBhGm/AABYF/GBxB3oISGPAABgTcQHEnpIAACwL+IDicQsGwAAbIv4QOIOPECYPAIAgDURH0hK1yGx3BAAACIYgYQxJAAAWEcgYWE0AACsI5C4eLgeAAC2EUgCPSR2mwEAQESL+EDipocEAADrIj6QuFipFQAA6wgkJV9ZGA0AAHsIJK7T7wMAAMKLQOJi6XgAAGwjkLAOCQAA1kV8IGGWDQAA9kV8IAkgjwAAYE/EBxI3034BALAu4gMJD9cDAMA+Aklwlo3lhgAAEMEIJPSQAABgHYGEWTYAAFhHIOFpvwAAWEcgYZYNAADWRXwgcTOGBAAA6yI+kAQwywYAAHsiPpCwdDwAAPZFfCAJjCEBAAD2EEhK8oifHhIAAKwhkARWamUQCQAA1kR8IHFzxwYAAOsiPpCwDgkAAPYRSEq+MoYEAAB7CCSBREIeAQDAGgJJYFArPSQAAFhDIGEMCQAA1kV8IOFZNgAA2BfxgYQeEgAA7COQsFIrAADWEUjoIQEAwLqIDySMIQEAwL6IDyTBHhLL7QAAIJIRSBhDAgCAdQQSxpAAAGAdgYQxJAAAWEcgoYcEAADrIj6QMMsGAAD7Ij6Q0EMCAIB9BJKSr8yyAQDAnogPJG56SAAAsC7iAwmzbAAAsI9AQg8JAADWEUhYqRUAAOsiPpAwhgQAAPsiPpAwhgQAAPvCGkj+9a9/6Uc/+pG6deumvn376uabb1ZeXl44T1llPO0XAAD7osN14OnTp2vWrFmaMGGC0tLSlJmZqVWrVsnn84XrlGck0EMiFfeSuMpuAAAA1SIsgSQjI0N/+ctfNG3aNA0aNCi4/YorrgjH6c5K2QBijDOgAACA6hGWWzZvv/22WrZs6Qgj56qyAYSZNgAA2BGWQLJhwwZ16tRJ06ZNU79+/dS1a1eNHj1aGzZsCMfpzor7hB4SAABQ/cJyy+bgwYPauHGjvvvuOz3yyCOKj4/Xiy++qPHjx+u9995To0aNzui40dGhz0+mTA9JVJQrLOeoSaKi3I6vkY56lKIWTtTDiXo4UY+qC0sgMcYoJydHzz//vDp37ixJ6tGjh4YOHarXX39dd999d5WP6Xa7lJRUN9RNVV5BUfDP9esnqE5c2Mb51igeT7ztJpxTqEcpauFEPZyohxP1qLywfPp6PB41aNAgGEYkqUGDBkpNTdXmzZvP6Jh+v5HXmxOqJgb5ytynOZJ5XPERHkiiotzyeOLl9ebK5/Pbbo511KMUtXCiHk7Uw4l6FPN44ivdSxSWT9+OHTtqx44dFb6Wn59/xsctKgrDX2qZWzaFhX7FREXuhVOWz+cPT71rKOpRilo4UQ8n6uFEPSovLDe3hgwZoqNHj+rrr78ObsvMzNSmTZvUpUuXcJzyjDnWIWF5NAAArAhLD8mll16qbt266a677tK9996ruLg4zZw5U7Gxsbr++uvDccozduI6JAAAoPqFpYfE7XZr5syZSktL05QpU3TfffcpMTFRf/vb39S4ceNwnPKMlV0HjXVIAACwI2wjOBs2bKg//vGP4Tp8yNBDAgCAfUyQVvGUYokn/gIAYAuBRFJJHpHfTyABAMAGAolKl4+ngwQAADsIJJJcJV0kDGoFAMAOAonK9pAQSAAAsIFAojJjSMgjAABYQSARs2wAALCNQKLStUiYZQMAgB0EEpXtIbHcEAAAIhSBRGXHkJBIAACwgUCi0ls25BEAAOwgkKj0lg09JAAA2EEgUZlBrQQSAACsIJBIiuKWDQAAVhFIJLl4uB4AAFYRSMTCaAAA2EYgUdkxJJYbAgBAhCKQSIqihwQAAKsIJCozhoRAAgCAFQQSsXQ8AAC2EUjEw/UAALCNQCLWIQEAwDYCiRhDAgCAbQQSsQ4JAAC2EUjEOiQAANhGIBHrkAAAYBuBRDzLBgAA2wgkktzMsgEAwCoCiSSXOzCGhEQCAIANBBKV9pAQSAAAsINAIm7ZAABgG4FEkrukCvSQAABgB4FEpeuQGGbZAABgBYFEpSu1kkcAALCDQCIGtQIAYBuBRAxqBQDANgKJJFdgUCv3bAAAsIJAojI9JCKQAABgA4FEZcaQ0EMCAIAVBBKVPlyPMSQAANhBIFHZab8kEgAAbCCQiFk2AADYRiBRaQ+JIZEAAGAFgUSlY0i4ZQMAgB0EEpXtIbHcEAAAIhSBREz7BQDANgKJGNQKAIBtBBIxhgQAANsIJGIdEgAAbCOQqMwtG7/lhgAAEKEIJCrTQ8LD9QAAsIJAIskV7CEhkAAAYAOBRJK7pArkEQAA7CCQqMw6JAxqBQDACgKJyq5DQiABAMAGAolKx5D4mWUDAIAVBBKVeZYNs2wAALCCQCLJHViplVGtAABYQSART/sFAMC2sAeS48ePKz09XcnJyfryyy/Dfboz4mKWDQAAVoU9kEybNk0+ny/cpzkr9JAAAGBXWAPJli1b9MYbb2jixInhPM1ZYwwJAAB2hTWQPP744xo9erTatWsXztOcNdYhAQDAruhwHXjRokX67rvv9MILL2jTpk0hOWZ0dOjzU1SUu/RZNmE6R00SFeV2fI101KMUtXCiHk7Uw4l6VF1YAklubq6efPJJ3XvvvUpMTAzJMd1ul5KS6obkWBUdW5Kio6PCdo6axuOJt92Ecwr1KEUtnKiHE/Vwoh6VF5ZAMn36dDVq1Eg//elPQ3ZMv9/I680J2fECoqLcwUCSl1+ozMzjIT9HTRIV5ZbHEy+vN1c+H0vXUo9S1MKJejhRDyfqUczjia90L1HIA8nu3bs1Z84cTZ06VVlZWZKknJyc4Nfjx4+rbt0z64UoKgrPX2pMSbGKivxhO0dN4/NRi7KoRylq4UQ9nKiHE/WovJAHkl27dqmwsFC33nprudfGjh2rHj166M033wz1ac9KdFRxD0kRs2wAALAi5IEkJSVFr732mmPb119/rSeeeEKPPfaYunXrFupTnrXAQNaiCO5WAwDAppAHEo/Ho759+1b4WpcuXdSlS5dQn/KsxZQEEp+PHhIAAGxgPpKk6Ch6SAAAsCls65CU1bdvX3377bfVcaozUhpI6CEBAMAGekhUesumyE8PCQAANhBIVNpDwhgSAADsIJCIMSQAANhGIBFjSAAAsI1AIik6unhhNJ/PzxN/AQCwgECi0qXjjSQ/gQQAgGpHIFHpLRuJ2zYAANhAIFHp0vGSIvqpjAAA2EIgkRTldgX/TA8JAADVj0AiyeVylT7xlx4SAACqHYGkBGuRAABgD4GkRBRrkQAAYA2BpAS3bAAAsIdAUiLaXfI8Gz89JAAAVDcCSYnA1F96SAAAqH4EkhKlt2zoIQEAoLoRSEoEZtmwMBoAANWPQFKCHhIAAOwhkJQIDGplDAkAANWPQFLCm1MgSfoi47DllgAAEHkIJCX2Hs6RJH38xV7LLQEAIPIQSAAAgHUEkhI3DetsuwkAAEQsAkmJ7NzC4J9z84sstgQAgMhDIClxyYUtg39+/7OdFlsCAEDkIZCUqBsfE/zzxowjFlsCAEDkIZBUYPPuY7abAABARCGQVCCt43m2mwAAQEQhkJTRv1szSVK7Fh7LLQEAILIQSMqIj4uWJOUX+Cy3BACAyEIgKaNObJQkKb+QQAIAQHUikJThUvETf7cwqBUAgGpFIClj5cbi59hs25dluSUAAEQWAkkZ/bo2s90EAAAiEoGkjPbN60uS2jarZ7klAABEFgJJGfXqFq/Weux4geWWAAAQWQgkZdRLiJUkZWblyxhjuTUAAEQOAkkZ9UsCiSTl5jP1FwCA6kIgKSMuNiq4Fok3h9s2AABUFwLJCTx1i3tJjmXnW24JAACRg0BygkAg8eYUWm4JAACRg0BygvqBQMJMGwAAqg2B5ATBWzYEEgAAqg2B5ASBmTb7Dh+33BIAACIHgeQEOflFkqTPvj1ouSUAAEQOAskJyo4dyckrstgSAAAiB4HkBKMvvSD453c+zrDYEgAAIgeB5ASeMqu1Nk1KsNgSAAAiB4HkFDZtPWK7CQAARAQCySl8vvmQDmTm2G4GAAC1HoHkNCbNWG27CQAA1HoEEgAAYB2BpAL3XtPD8X2Rz2+pJQAARAYCSQUS6kQ7vn/3462WWgIAQGQgkFSgXXOP4/v5q7bLGGOpNQAA1H4Ekgq4XS5NGNnFsW3x2l2WWgMAQO1HIDmJPilN1alVg+D3b3zwvTZsPkRPCQAAYUAgOYW+KU0c3z8/7wv9/Kmlys3nGTcAAIQSgeQU+qY2q3D7G+9/V80tAQCgdgtLIFm4cKFuv/12paenKy0tTSNHjtS8efNq3O2O+LioCrev2LivmlsCAEDtFpZA8sorryg+Pl6TJk3S9OnTlZ6ersmTJ2vq1KnhOF3YuFwuPXTDhbabAQBArRd9+l2qbvr06WrYsGHw+379+uno0aN6+eWX9Ytf/EJud825U9S+hafC7S++u1Fjr+isrNwC1a0To8T4mGpuGQAAtUdYAknZMBKQkpKiN998Uzk5OUpMTAzHacPC5XLpget6an9mjnpe0Fj3vPCxJOmTrw/ok68PBPebM2morSYCAFDjhSWQVGTt2rVq2rTpWYWR6OjQ96xERbkdXyvStUMjdVUj+U8xBublBV/roy/26rm7Bqihp07I21ldKlOPSEI9SlELJ+rhRD2cqEfVuUw1jDT97LPPdMMNN+jBBx/UTTfddEbHMMbI5XKFtmFn4Opfvnvaff7zzMhqaAkAALVH2APJvn37NGrUKHXo0EFz5sw54/EjPp9fXm9uiFtXnF49nnh5vbnyVeIhet/uOKrfvfbZKfeZ86uhiq6hqbiq9ajtqEcpauFEPZyohxP1KObxxFe6lyist2y8Xq9uueUWNWjQQC+88MJZD2YtKgrfX6rP56/U8TucZJBrWa8u/EaHjuXpiotaqWv7RqFoXrWrbD0iBfUoRS2cqIcT9XCiHpUXtkCSl5en2267TVlZWfrnP/+pevXqhetU1W7G/YO0dW+WOpzv0S1/WFbu9Q8/3yNJ2rT1iGbcP0gx0VHy+41y8ouYjQMAQAXCcl+hqKhI99xzjzIyMjR79mw1bdo0HKexJiY6Sp1aNVCU263WTU49SPe2pz/U+u8O6vX3v9Ndz3+kaf/6sppaCQBAzRGWQPLYY49p6dKlmjBhgrKzs/X5558H/ysoKAjHKa15ZNxFp93nhbe/1LL1uyVJn317UOOfXKI3l24Od9MAAKgxwnLLZsWKFZKkJ598stxrixcvVsuWLcNxWitcLpcu7d1SH3y2S26X65RTg8tatGaHrurXRnExUTV2ACwAAKESlkCyZMmScBz2nHX9pZ30k/T22nkgW0+8vq7SPzfxuY8kSdddeoEu691K+QU+bdp2RNv3ZWnkgHZyu+1PcwYAoDpU28JotV2d2Gh1PL9+8Pvz6tfRw2N7B1d2PZW/f/C9/v7B945t/1m5TZdf1ErXDu14Tqy/AgBAOHGvIIRcLpcG9zxfLpd0x4+7yVM3VlNu6q1GnrgzOt57n+7Uz59aqqXrdmn/kZzg9qycAnmP166xOACAyFYtK7WGgs/n15Ejx0N+3Ohot5KS6ioz83hI5or7jVFOXsXTe8c/eXa3sibf2FvNGibojmeXS5Km3puu+LjynVxFPr+2789S++YePfH6OqX3aKEB3ZtX6hyhrkdNRz1KUQsn6uFEPZyoR7GGDeueGwujRSK3y3XStUbGX5miOQu+1mW9W+n9z3YGB8NW1m9fda4Q+93Oo+rR8TxJ0kv//UorNu7To+Mu0qMvf+rYb/PuY+qb2lQxYXgWEAAAoUAPyTmQYr/adkRP/+PzM/75Nk3rafv+rFPu85P09hr+g7anPda5UI9zCfUoRS2cqIcT9XCiHsXoIalhUts21JxJQ5WTV6Q9h44rOtql37xy6ufllHW6MCJJby/PUJ/UpmrSIP5smgoAQFgQSM4hCXWi1bFl8UydOZOGKr/Ap7jYqLMeexIw6cVVwT+3bJyoH6e3U88LGjv2uX/qCh3IzNWrD10SknMCAFAZDCo4h8XFRkmS/nz3QF3Vr41+2Ld1yI6962C2XnjrS634cq8OHS1+inJhkV8HMov//Kd/bpAk/fbVTzXxueWOBd/yC3z687wv9PEXe0PWHgBAZKOHpAZIjI/RTwd1kCRdM6SjjDF6/LXPtHXv6W/VnM5L87+ucPuGzYccPTN7Dh5Xs0YJen7eFzqWXaBdB7P1+eZD6t25serEll5G2bmFSoiL1sI127Xn0HFt3n1Mj43v49gHAIATMai1hg48Msbo508tlSQl1YtTZla+JCmlTZJ+OTpNN5e8Vh3+fPdAZezx6rm5G066zzVDOmpA9+bBGUhHvHny1I097bL5R7x5SqoXZ21xuJp6fYQDtXCiHk7Uw4l6FGNQawRwuVx66cEhOnwsTw09dbTokx1KbtVAHcqsFhvQ0BOnI9784PcPjb1Qv3ttbcjasnzDHs1btuWU+7y5dLPeXLpZcyYN1bc7MvXUG+vVqVUDTfpZr5P+zMx/b9Lqr/ZLKh5TAwCoveghqaUptuztljmThmr1V/u068Bx/XRQe7lcLm3edUy/fz10oaSyOrasr827jpXb/ocJ/bTjQLbat/Dovr+sUN060TqeV+TYZ0jP8zWge3MtXrtLwy5uo8mz10gqDStFPr+Wb9ijgd1bhGzNldp6fZwJauFEPZyohxP1KEYPCXRVvzaav2p78PuLU5tJqaWvNzxhOfuRA9rph31b67NvD+il/1Y8riQUKgojkvRAmRlAksqFEUlaun63lq7fLUlauXFfcHtmVr6+3ZGpmf/5SpL0+nvfqVnDBP3+1ouD++w9fFyeurGqW6fiResAAHYRSGqpH6e3V0qbJHVoUf4WjlQ87qRvalNt2X1MT03oFxyj0atTY72kUweSqfem6/X3vtWqTftD3u4z8cupK8pt23ckR+OfXKIfD2ynxIRY/fV/30qSpt83SHGxUTp2vED3lnnwYfcOjXTPqB7an5mj/6zYpjGXdzrpQNzn525Qi8Z1NWpwx/C8IQCIQNyyoVvNITrarT2ZeTp0JFupbRoGt8+Z/7U+/nKvHh13kVo3rSep8s/miY5yqch3blxmnVs30APX96qw7SMHtNO7H28Nfj9n0tDg9XH4SLZu+t1ix/6zHhisKHdxV+QXWw7p3Y+36e5R3XXoaJ6aNYxXwkl6Y/ILfPL5/UqoE6P8Ap+iolynHdx7LuDfihP1cKIeTtSjWFVu2RBIuGgcqlKPgkKfJjzzYfD7OZOGOmb/SFK75vU0+caLtD8zR7+asTq4/ZGbLtJr//tWW/d6Q/8mQuzua3vq+X+uL7e9bp1o/fEXP1BcTJTjPZf1pzv7660Pt+jqH7RVk6QEbdp2RM+UPCbgyQn9NHn2GjVJitdvf97X8XOBOnZu3UD3X9dTR7Py1dBTJ+TvrSr4t+JEPZyohxP1KEYgqQIuGqeq1sN7vEDL1u/WD/u2VmxM8UJux7LzlV/kL7dMfW5+kfYdyVHbZvWCt4h8fr/mLt2i9z7dGdxvwsgu6pPSVPszcxQfF60d+7L0pzdPPqW4pmjeKEF7D+dU+FrXdg11zzU9tOfgcU2Z84njtfQezbV8w16NubyThvZqKak4sMxdukWLPtmhO3/STb06Fa+4u+LLvfr3iq16dFwfFRT6JEn1E53jhaTiMTW7Dh7X9Hc2atjFrSt1+4l/K07Uw4l6OFGPYgSSKuCicbJVD7/fyO0++VojgVssV/Vro58O6iBjjPIKfKoTG6XFa3fpjQ++D+57+UWtHAHnVPqkNNEnXx84u8ZXozmThqqwyKfbnv6w3GuP3HSRHnvl03Lbp9zUWyu+3KefpLdXfFy0tu71lntydPNGCZowsqtaNUl0bDfG6Gh2gZLqxfFv5QTUw4l6OFGPYsyyQY1zqjAiSb/5eR9tzDiiy/u0klS8Dkt8XPHle2nvVo5AMvqSC5RX4NPyDXskSdddeoEOHc3T+5+VDykTRnZVy8bb9PbyjFC9lbDKyinQ3X/+uMLXKgojkoIPaly8dpdcLqmi/wXZezhHj8z5JDjoN6Dsragot0s+v9ErZ/Gco2PZ+TqSla+mSQl64vW16tahka4ZwuBgAAQS1BAtGyeqZePE0+9Y4qZhnXXTsM7B74t8fu09fFw7D2brFz/qqo7n1w/eNrqqXxstWL1deQU+PTy2txLqROvXM1dXeNzf3txX2TkF+np7pv69Ylu51wd2by5jpI+/DM9zfk4WRirrdP2ht/+puOdl7BXJSk9r4XjN5y/+4YdmrNb91/XU/VNXyOc3mnpvejAcSpI3p0C7D2QruU1ScMXg2Q8Mkcsl3fsX54yo3YeK18YJDA4+E4FxOSP6t9WPBrZ3vOY9XqCvth3Rmq/265arU0860BiAfdyyoVvNoabW46FZq4PjM0K1qmthkU/Pz/tC467uqsaeWPlPmCn01bYj2rLHq3+V9K48PLa32rfwSFK5wb0BNw9PUaeWDZSYEKNf/Gm5pOJbKoFejJpq5v8NVpTbpdz8It353EflXk/reJ5u/GFyuUASEPg7233ouLbu8WrOguKp53f9v+56ZeE3umV4qlo1TZQnIVZS8S0+vzFaum63/r74e8ex+ndtphUb92nU4A6ae8IKwlW5NjZuPazGDeKVX+BTk6T4ctPAa+q/lXChHk7UoxhjSKqAi8apJtfDb4zcIX7mTWXqsWN/lop8JhhGyjoxmJT9QCwo9OnY8QI1bhCvzKx8x3oqz9zRX1k5BXr05YpvwwT069JMt1ydWukp2LVB2Wc3VdVLDw7Rjv3ZapIU7+jVCXh+7gZt2HJYbZvV07Z9zodX/ubnfdSycaKMMcrY41Wb5vXUrIlHR4/mBK+NrJwCbcw4ot6dGysmuvjW1/8+2aFdB7M1bliK9mfmaMHq7brh8uTgIPDaoib/7ggH6lGMMSSISKEOI5UVWJelIi6XS62bJmrH/mxdUjJDJiA2JkqNS2YiJdWL00sPDgkO1HW5XEqqV352zIluHp5Sbtuf7uyv+0p6Ih664UIdOJqrXhc0Dt6OCZhx/2AVFPn0+nvfac1X58Yid5VxpmFEklZ/tV+z/vOVYmPcevGXg7V47S6t/faA0jqepx0HsrVhy2FJKhdGJGnKS5+U2xbwy9Fpat/cE7yl9uHn9XXz1al6YHrpCsRRbndwXNOKL/cFw+nBo7nFg4aj3DLGFN9Wc9m7ngFb6CEhxTpQD6dQ1KPI59eW3cfU4fz6VV4AzW+MduzPUsvGidpz6LjmLtuiTVuPBF8PfKjl5hfpjmeXq8P5Hj10Q28dzyuUz2fkqRvrOF5ghk0jTx398Rc/CG4/XQ/LHT/uqn1HcvTWhzVj8G9ldGmbpE3bMkN2vNZNErXjQHal93903EV6/f3vgo9TmDNpqG794zIV+Yqvs3bNPXpo7IWOYFLk8+vWPy6TVHyb7FTXkzFGB4/mqqDIX6XxV6dzsp5Ifnc4UY9i3LKpAi4aJ+rhdC7WY+7SzVq4Zof+77qeSmmTFJJj+o1RXr5P8XHFtxGMFByQ+vOrUtS/W3PlF/l0e5npxj+/KkUvzQ/fc48gXf2DtmpxXl3N/M8m/f7Wix2LC156YUtdf1kn5eYXadPWI5r2zkZdd8kF5cbUBIy+5AJdflGr4Pf5BT59/OVe9U5urPqJcTLG6PX3vlODxFhd3b9dcB9Jiol2a/v+rOB08T/fPVBHvHnKzS9ScuviazA62q369RM06S8faWivlrowuXG5NmzMOKxFn+zQjT/srEb165y0F6iwyKdj2cWDx+PjotW7c5MzqJ5d5+LvDhsIJFXAReNEPZzO1XoYY4KzhMKlsMivo9n5wdtK0dFubd1/XLk5+erUqoEkaerbX2rtdwclOR/oOOP+QcExFF9mHNazJQvbPXh9T7Vqkhgc+DrrgcH6xweb9eGG3WrVJFHJrZK06JMdatk4Ud06NNTC1Ts0uOf5GntF8kl7cX5+VYouTG4cHCSMk2vWMEGPjLtI/1uzQ++UeUyCVDwz7ZWF30iSLkxurLXfHqzUMeNionTDFZ307c6j+miDc3ZZjw6N1L1DI23dm6VxV3Z2jKeqlxCj34zvo/qJcVq8dpeOZufrJ+nFTyO/7ellKizz763sYxpOJje/SBl7vUppnVThMgJ+v9HBY7lqmpRQ4c8XFPpCOq7nXP3dUd0IJFXAReNEPZyoR6mKamGM0ZJ1u9W7cxPVS4jRhu8PqW1zT7nxL8/N3aCDR3P1u1surujQQcYYHT6Wp0b165QLXMYYbdnj1e//ulbJrRqooSdODerFBVeZzc4tlL/kNkXDenX098Xf6+DRXE0e21v/XbVN73y0taJTnlSbZvW0fV+WmjdK0M3DU8stJofqc/uPumr6OxvVtGGCHr3pIsdaOQFlA+sVfVrpmiEdHdfQnAVf6+Mv9iq9RwvHkgCSNG/ZFi1YXRymZz8wRM/N26COLeprxIDinqLCIp+27PaqY8v6uu3pZbry4uIFGk+062C26taJCdlCgu9/tlMZe7y6bUQXx3ZjjIp8/mDoP5cRSKqADxwn6uFEPUrV9Fr4/UZ5BeWnJddLiNE9o3rolYXfaGeZMSDP3tlfnrqxwQ+1HfuzHLOezj+vribffLEmPOl86OLpXHfpBVrz1X5l7Dn3n+NUEwzueb6Wrd9d4Wt3/7/uen7eF3p03EXlZqzNmTRUPr9fv3ttbYWDmMvuV1HvXGC2V/cOjfTFlsN68PqeeuqN4mdePTdxgDZtO6JBvVvLX1ikoiK/Cov82rzrqDq2bKCY6PIf0Cf20Lyy8JvgIOhxwzprYI/SdYEemL5Sh47laeSAdvLUjdWQnueXO96mrUe05/BxXda7VbnXqhOBpApq+i/ZUKMeTtSjVG2pRVZOgZZv2KNLL2yl6GhX8FbAgcwcvfvxNrVtVk/NGiWoW/tG5X727eVb9N+Vxf8n/drDlyopqa7+On+T3lyyWZN+1kutmyYqv9Cve18onm0zblhnLV2/W1de3EbT3tmoS3u31PWXdpJUPG7nnY+2KievUEvWFX+g/t91PbV0/W7ViYk6q8X1+nVpplWb9p3xzyN0urZvqI0ZRyp8bWD35rppWGe98NaX+nzzIbVukqi0C87TgG7N9cCLqxz7BlZR9hsTHN8VcM+o7ure4bzg997jBbqn5Brs1r6RJozsoqXrd2tQWgvVPWFxwKPZ+crOLdSUlz5RzwvO08Sfdg/F2w4ikFRBbfklGyrUw4l6lKIWxbNcvthyWJ1bN5AnMe6k9Sjy+ZVf6Cv3y/9k/MbI7zflZs1U9H/mNw9P0Q+6Ntfkl9Zo98GKfye+9OCQYM/OqWZQtWvu0a/G9ArO3DnRc3cN0D0lU5ndLpfaNa+nC5Ob6M2lmyvztmBBI08dHfbmnfT1Zg0TdFHnJuqT2lSTZ68p93qoFpYMIJBUAb9knaiHE/UoRS2cqqMeuflF2rrXq84nGaj5zfZMNapfR40bxMvvN/r3iq1KbtVAKW0bBvc5dDRXS9fv1rCL26hunWi5XC5t35elw9684FOiy8rJK9I3OzLVrX2j4K2F/AKfY9xGRSHnobEXKrZOrNqcl6Cn/75e9RNjtf9IrjbvPubY7/zz6mr3odP/Lh/Rv22Fj2dAeBFIKoFAUj2ohxP1KEUtnCK5Hs/N3aAvthzWQ2MvVNbxQqW2TVJCfEyF9Zi/altw/ZqKPuyKfH5l7PGqfQuPCov8uuPZ5cF9/cboN698qh37i8f2vPTgEM37cIuOZuVr1aaKF/O795oeSkqM05Q5FS9k98wd/R2rIp/Ms3f2L/eog9tGdNGMf2867c/WZDYDCSu1AgCq5J5RPSq97+UXtVZMlFvdOpQfkyNJ0VHu4DTy6Ci34wPR7XLp0XF99N3Oo2rWKEEulys4q6peQqwS4qI1YkC7YI/NgO7Ng2N/Jt/YW0+8vk6tmiRq697iAcT/NzpNSfXilNo2SV+dZFG8H3RtpusuvUB168SUG9DaN7WpFq7eXukF8Nq18GjSz3opv8CnvIIiuV0u3T9tZaV+VpIu6dVSi9ftqvT+J2reKCH4jK/KGD3U7pO36SGJ4P/LqQj1cKIepaiFE/VwslmPo9n5OpCZGww2lbX38HEtXb9bH3y2SwO7N9eFyU3UrX1Dx3ThwiK/Vm3ap4tTmwZnwWzfl6XHXvlUSfXi9Oi4i4KPDBg1pIPmLi1+oGNg0PPJ6mGMUXZuoeMJ3nMmDdUbH3yn/AKfOrdOUp/UJtq2L0vvLM/QiAHt9Ok3B/TBZ7sc68T8dFD7YA/U+efVVd/Upnp7eYb+cHs/nVc/Xh99sUcvL/hGg9JaqH7dWH2++VCwxylwzodmrVZOfpGeuq1fyJ+xxC2bKuCXihP1cKIepaiFE/VwirR65Bf4FBvjlsvlUpHPryi3S35j9MlXB9SpVQM1bZRQqXrsOpCtKXM+0YBuzTX+qvLPpqpI2UcIBBaNq8piiZ99c0A5+UVKLzOVOFyLLRJIqiDS/hGdDvVwoh6lqIUT9XCiHk7hrkd1rNYcClUJJFV70hcAALCuJoSRqiKQAAAA6wgkAADAOgIJAACwjkACAACsI5AAAADrCCQAAMA6AgkAALCOQAIAAKwjkAAAAOsIJAAAwDoCCQAAsI5AAgAArCOQAAAA61zGGGO7EZVhjJHfH56mRkW55fPxuOwA6uFEPUpRCyfq4UQ9nKiH5Ha7Kv1k4hoTSAAAQO3FLRsAAGAdgQQAAFhHIAEAANYRSAAAgHUEEgAAYB2BBAAAWEcgAQAA1hFIAACAdQQSAABgHYEEAABYRyABAADWEUgAAIB1BBIAAGBdxAaSLVu2aNy4cUpLS1P//v31hz/8QQUFBbabdVbefvttJScnl/vv6aefduw3d+5cXXHFFerWrZtGjBihpUuXljtWVlaWfv3rX6tPnz7q2bOn7rrrLh04cKDcfuvWrdO1116r7t27a8iQIZo5c6ZsPEB6+/btmjJlikaOHKnU1FQNHz68wv2q+70bYzRz5kwNHjxY3bt317XXXqvPP/88JO/5VCpTjxtuuKHC62XLli2O/Wp6PRYuXKjbb79d6enpSktL08iRIzVv3rxybYuUa6My9YiUa0OSPvzwQ40ZM0YXX3yxunbtqksuuURPPPGEsrKyHPstWbJEI0aMULdu3XTFFVforbfeKnesgoICPfXUU+rfv7/S0tI0btw4ZWRklNuvsp8/lbkmaxUTgY4ePWr69+9vfvazn5nly5ebuXPnmgsvvNA89thjtpt2Vt566y3TqVMns3z5crN+/frgf3v27Anu89///tckJyebZ5991qxatcpMnjzZpKammvXr1zuONX78eJOenm7mz59vPvjgAzN8+HAzYsQIU1hYGNxn27ZtJi0tzdxxxx1m5cqV5uWXXzZdunQxs2fPrq63HPT++++b9PR0M3HiRDN8+HBz1VVXldvHxnufMWOG6dKli3n55ZfNypUrzR133GF69uxpduzYEZY6BFSmHmPGjDGjR492XCvr1683eXl5jv1qej2uueYac++995r58+eblStXmqefftp07tzZvPDCC8F9IunaqEw9IuXaMMaYd955xzz11FNm0aJFZvXq1eavf/2r6dOnjxk3blxwn08//dSkpKSYyZMnm1WrVplnn33WJCcnm4ULFzqONXnyZHPhhReauXPnmuXLl5vrr7/eDBw40Hi93uA+lf38qew1WZtEZCB58cUXTVpamsnMzAxu+8c//mFSUlLMvn377DXsLAUCyeHDh0+6z+WXX27uu+8+x7Zrr73W3HzzzcHv161bZzp16mQ++uij4LYtW7aY5ORkM3/+/OC2yZMnmyFDhpj8/Pzgtmeeecb07t3bsa06+Hy+4J8ffPDBCj+Aq/u95+XlmV69eplnnnkmuE9+fr4ZMmSIeeSRR878zVZCZeoxZswYc+utt57yOLWhHhX9e3j44YdNr169gnWKpGujMvWIlGvjZP75z3+aTp06BT8Pxo8fb6699lrHPvfdd58ZNmxY8Pu9e/ealJQU849//CO4LTMz06SlpZmZM2cGt1X286cy12RtE5G3bJYvX65+/fqpQYMGwW3Dhg2T3+/XihUr7DUszHbu3Klt27Zp2LBhju1XXnmlVq1aFewyXL58uTwej/r37x/cp3379kpJSdHy5cuD25YvX65LLrlEsbGxjmN5vV6tX78+zO/Gye0+9aVs472vW7dO2dnZjnPGxsbqsssucxwrHE5Xj8qqDfVo2LBhuW0pKSnKzs5WTk5OxF0bp6tHZdWWelQk8NlQWFiogoICrVmzRj/84Q8d+1x55ZXasmWLdu3aJUn6+OOP5ff7Hfs1aNBA/fv3L1eP033+VPaarG0iMpBkZGSoffv2jm0ej0eNGzeu8H5fTTN8+HClpKTokksu0YwZM+Tz+SQp+N7atWvn2L9Dhw4qLCzUzp07g/u1a9dOLpfLsV/79u2Dx8jJydHevXvL1bF9+/ZyuVznXB1tvPfA1xP369Chg/bs2aO8vLwQvbsz98knnygtLU3dunXTmDFj9Omnnzper631WLt2rZo2barExESuDTnrERBp14bP51N+fr42bdqkqVOnaujQoWrZsqV27NihwsLCCttWtu0ZGRlq1KiR6tevX26/sr8PK/P5U9lrsraJtt0AG7xerzweT7nt9evX17Fjxyy0KDQaN26siRMnqkePHnK5XFqyZImee+457d+/X1OmTAm+txPfe+D7wOter1f16tUrd/z69etr48aNkhQc8HXisWJjYxUfH3/O1dHGe/d6vYqNjVVcXFy5cxpjdOzYMdWpU+ds39oZu+iiizRy5Ei1bdtWBw4c0EsvvaRx48bpr3/9q3r27Cmpdtbjs88+04IFC/Tggw9K4to4sR5SZF4bQ4YM0f79+yVJAwcO1DPPPCPp7K8Pj8fj+H1Ymc+fyp6ztonIQFJbDRw4UAMHDgx+P2DAAMXFxenVV1/VhAkTLLYM56K77rrL8f3gwYM1fPhwTZs2TbNmzbLUqvDat2+f7r33XvXt21djx4613RzrTlaPSLw2Zs6cqdzcXG3evFnTp0/XhAkT9PLLL9tuVkSJyFs2Ho+n3JQuqTh1ntjdVtMNGzZMPp9PX3/9dfC9nfjevV6vJAVf93g8ys7OLnessvUJ/J/AiccqKChQbm7uOVdHG+/d4/GooKBA+fn55c7pcrnOuRolJCRo0KBB2rRpU3BbbaqH1+vVLbfcogYNGuiFF14IjrOJ1GvjZPWoSG2/NiSpc+fO6tmzp0aNGqVp06ZpzZo1ev/998/6+vB6vY72V+bzp7LnrG0iMpCUvccZkJWVpYMHD5a7t1ebBN7bie89IyNDMTExatWqVXC/rVu3llsnYOvWrcFjJCQkqHnz5uWOFfi5c62ONt574OvWrVvLnbNFixZWb9dUVm2pR15enm677TZlZWVp9uzZjq71SLw2TlWPyqpN9ThRcnKyYmJitGPHDrVu3VoxMTEVXh+SHO/h0KFD5W6nnDhmpDKfP5W9JmubiAwk6enpWrlyZTBtStKiRYvkdrsdI8ZrgwULFigqKkqpqalq1aqV2rZtq0WLFpXbp1+/fsFR8Onp6Tp27JhWrVoV3Gfr1q366quvlJ6eHtyWnp6uxYsXq7Cw0HEsj8cTvM98rrDx3nv16qXExEQtXLgwuE9hYaHee+89x7HOFTk5OVq2bJm6desW3FYb6lFUVKR77rlHGRkZmj17tpo2bep4PdKujdPVoyK19do4mQ0bNqiwsFAtW7ZUbGys+vbtq//973+OfRYsWKAOHTqoZcuWkopvkbvdbr333nvBfY4dO6aPP/64XD1O9/lT2Wuy1rEx19i2wMI0Y8aMMR999JGZN2+e6d27d41fGG38+PFmxowZZtmyZWbZsmVm8uTJJjk52fzud78L7vOf//zHJCcnm+eff96sXr3aTJkyxaSmppp169aVO9agQYPMggULzOLFi0+54NHEiRPNypUrzSuvvGJtYbScnByzcOFCs3DhQjNmzBgzaNCg4PeBdRdsvPcZM2aYrl27mldeecWsXLnSTJw4sVoWezpdPT799FNz2223mXnz5plVq1aZd9991/zoRz8yXbp0MRs2bKhV9Xj44YdNp06dzJw5c8ot9BVYByOSro3T1SOSrg1jjLnjjjvM9OnTzZIlS8zKlSvNnDlzTP/+/c3VV18dvD4CC6M98sgjZvXq1eb55583ycnJZsGCBY5jTZ482fTu3dvMmzfPfPTRR2bMmDEnXRjtdJ8/lb0ma5OIDCTGGLN582Zz4403mu7du5t+/fqZJ598stoX8wq13/72t+byyy833bt3N127djXDhw83r776qvH7/Y793nzzTXPZZZeZLl26mOHDh5slS5aUO5bX6zW/+tWvTO/evU1aWpq58847K1w0bu3atWbUqFGma9euJj093cyYMaPc+arDzp07TadOnSr8b/Xq1cH9qvu9+/1+8+KLL5r09HTTtWtXM2rUqGr5hXK6emzbts2MHz/e9O/f33Tp0sX07t3b3HLLLeU+cIyp+fUYMmTISWuxc+fO4H6Rcm2crh6RdG0YUxyERo4caXr27GnS0tLMVVddZZ577jmTlZXl2C+wEm2XLl3MZZddZubOnVvuWPn5+ebJJ580/fr1M927dzc33XST2bx5c7n9Kvv5U5lrsjZxGWPhwSMAAABlROQYEgAAcG4hkAAAAOsIJAAAwDoCCQAAsI5AAgAArCOQAAAA6wgkAADAOgIJAACwjkACAACsI5AAAADrCCQAAMC6/w+LaKyjW47QjAAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "77CgIkYjnEXk"
      },
      "source": [
        "# Manual Testing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XoZhIC8U7_GV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a8066bba-a8f4-4756-957b-6f21a21b9997"
      },
      "source": [
        "test_sentence = 'Where do you live?'\n",
        "print('Raw input text:', test_sentence)\n",
        "\n",
        "input_ids = encode_input_str(\n",
        "    text = test_sentence,\n",
        "    target_lang = 'bn',\n",
        "    tokenizer = tokenizer,\n",
        "    seq_len = model.config.max_length,\n",
        "    lang_token_map = LANG_TOKEN_MAPPING)\n",
        "input_ids = input_ids.unsqueeze(0).cuda()\n",
        "\n",
        "print('Truncated input text:', tokenizer.convert_tokens_to_string(\n",
        "    tokenizer.convert_ids_to_tokens(input_ids[0])))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Raw input text: Where do you live?\n",
            "Truncated input text: <hi> Where do you live?</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "44BvtXAOHnWL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "18a7187f-d074-4b21-ff76-3f80ae5a263d"
      },
      "source": [
        "output_tokens = model.generate(input_ids, num_beams=10, num_return_sequences=3)\n",
        "# print(output_tokens)\n",
        "for token_set in output_tokens:\n",
        "  print(tokenizer.decode(token_set, skip_special_tokens=True))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "तुम कहाँ रहते हो?\n",
            "आप कहाँ रहते हैं?\n",
            "तुम कहाँ हो?\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "AhijXoO9VBeX"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}