sequence_labeling_mlp.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "3b7e4bb9a3334cf3b0d7a09629c5d703": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d641e90a212744199f7d98af7791e79e",
              "IPY_MODEL_fed0221314094363a37114517e917dcf",
              "IPY_MODEL_18e0eb340f5d4a8da8fad50c21f8161e"
            ],
            "layout": "IPY_MODEL_198f8ea06b2a4fae89fc5449c1fc61a6"
          }
        },
        "d641e90a212744199f7d98af7791e79e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_21a3f45ce81f4430bfa0dfda3969d9d0",
            "placeholder": "​",
            "style": "IPY_MODEL_8db7b271b6024c5dbf9c17ec222dd93a",
            "value": "Downloading data: 100%"
          }
        },
        "fed0221314094363a37114517e917dcf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5bb9cded71ae4cc5901e27fe1f9b7566",
            "max": 1227788,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_622e5f4f4ed9471fa78a6652eb1b9812",
            "value": 1227788
          }
        },
        "18e0eb340f5d4a8da8fad50c21f8161e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_874542c532694e1fa4cc46b91f9edb48",
            "placeholder": "​",
            "style": "IPY_MODEL_19fc9de7cfb9483b85b20f11b8871f31",
            "value": " 1.23M/1.23M [00:00&lt;00:00, 2.03MB/s]"
          }
        },
        "198f8ea06b2a4fae89fc5449c1fc61a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "21a3f45ce81f4430bfa0dfda3969d9d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8db7b271b6024c5dbf9c17ec222dd93a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5bb9cded71ae4cc5901e27fe1f9b7566": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "622e5f4f4ed9471fa78a6652eb1b9812": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "874542c532694e1fa4cc46b91f9edb48": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "19fc9de7cfb9483b85b20f11b8871f31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fd65ed9d7f8e43c083334c70096ab97f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_69c38d47a26d45cfb977e77f963d571e",
              "IPY_MODEL_7ec7d73c10ea4605b756cdc888987a8d",
              "IPY_MODEL_654e236e93cd467a98e28a2aca9f2feb"
            ],
            "layout": "IPY_MODEL_60690be8d9ee41769a5cbdb7d6950c92"
          }
        },
        "69c38d47a26d45cfb977e77f963d571e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_28e4758925884be698ee23806cdda147",
            "placeholder": "​",
            "style": "IPY_MODEL_291dc23cd40f4f6094c25854f8851bbd",
            "value": "Downloading data: 100%"
          }
        },
        "7ec7d73c10ea4605b756cdc888987a8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bb3c99c42fd14e339e7c07b9b1a88e25",
            "max": 311883,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f1f90b6362c54e9f835c40ff50c54fef",
            "value": 311883
          }
        },
        "654e236e93cd467a98e28a2aca9f2feb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9d1d0c33523945d69667e45dbb67d536",
            "placeholder": "​",
            "style": "IPY_MODEL_6c202e42e2414cd4a5f871466cb1391d",
            "value": " 312k/312k [00:00&lt;00:00, 1.06MB/s]"
          }
        },
        "60690be8d9ee41769a5cbdb7d6950c92": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "28e4758925884be698ee23806cdda147": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "291dc23cd40f4f6094c25854f8851bbd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bb3c99c42fd14e339e7c07b9b1a88e25": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f1f90b6362c54e9f835c40ff50c54fef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9d1d0c33523945d69667e45dbb67d536": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6c202e42e2414cd4a5f871466cb1391d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "773d0c2f62c74a3296538a664e507b3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_43882e01df1c41469fed318c2b525ada",
              "IPY_MODEL_e6013f778a6e4851ae56acffbd4e0d74",
              "IPY_MODEL_ca7893865fff415f96b042c6cbd5904c"
            ],
            "layout": "IPY_MODEL_e85560080b6c47d7a6169dfd76389ecf"
          }
        },
        "43882e01df1c41469fed318c2b525ada": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_436144cba76d4eb69d2f97d8cd9d49c1",
            "placeholder": "​",
            "style": "IPY_MODEL_a58c9522f7584b3ca1c9b053596a1646",
            "value": "Downloading data: 100%"
          }
        },
        "e6013f778a6e4851ae56acffbd4e0d74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5558da787ca240d4a90cc924f15b9f8b",
            "max": 283307,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a8530e4977094ee2b8ba29de3778e6ce",
            "value": 283307
          }
        },
        "ca7893865fff415f96b042c6cbd5904c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_181e26da59174240a9999520ee035a5c",
            "placeholder": "​",
            "style": "IPY_MODEL_16acb247a1ff4fd0b362cfd08b0f23ec",
            "value": " 283k/283k [00:00&lt;00:00, 845kB/s]"
          }
        },
        "e85560080b6c47d7a6169dfd76389ecf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "436144cba76d4eb69d2f97d8cd9d49c1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a58c9522f7584b3ca1c9b053596a1646": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5558da787ca240d4a90cc924f15b9f8b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a8530e4977094ee2b8ba29de3778e6ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "181e26da59174240a9999520ee035a5c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "16acb247a1ff4fd0b362cfd08b0f23ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "10d247583d8b45188a8cf1c8554a4e33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_123fe0647fd84174ae030017d4dcbf6c",
              "IPY_MODEL_6aa220e9a81d4d1296de9f91926246b4",
              "IPY_MODEL_27d6857a0f5b4301a9c4fec5325960d2"
            ],
            "layout": "IPY_MODEL_da99f48993ee4c168b2076ae7a22861f"
          }
        },
        "123fe0647fd84174ae030017d4dcbf6c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9e351ef8fd2b4710ad73ba9aa2fbc4a2",
            "placeholder": "​",
            "style": "IPY_MODEL_e07368c90ef548aa8e54e3106b9df600",
            "value": "Generating train split: 100%"
          }
        },
        "6aa220e9a81d4d1296de9f91926246b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c5c91ac98d244093a4a7c20ae82d2759",
            "max": 14041,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ca1e23759298416b939ebb8b3e1d5538",
            "value": 14041
          }
        },
        "27d6857a0f5b4301a9c4fec5325960d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_16ce1d1f13ce4dfa9b3279cfb9e4623a",
            "placeholder": "​",
            "style": "IPY_MODEL_51d00181226b4b949519f76583c1dd78",
            "value": " 14041/14041 [00:00&lt;00:00, 9898.23 examples/s]"
          }
        },
        "da99f48993ee4c168b2076ae7a22861f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9e351ef8fd2b4710ad73ba9aa2fbc4a2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e07368c90ef548aa8e54e3106b9df600": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c5c91ac98d244093a4a7c20ae82d2759": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ca1e23759298416b939ebb8b3e1d5538": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "16ce1d1f13ce4dfa9b3279cfb9e4623a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "51d00181226b4b949519f76583c1dd78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d94f7c6b4c874299854dfa90495ad550": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b43511af23c2499bb841866a5a68f1ef",
              "IPY_MODEL_6f559852c65246d095e3e215c5a9822f",
              "IPY_MODEL_15559040fe784a788f7b22318db88c7f"
            ],
            "layout": "IPY_MODEL_4bd9daf59897421dbc159f10f7bf5443"
          }
        },
        "b43511af23c2499bb841866a5a68f1ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d28a065b2ea349ff9f7ea8424aa1f262",
            "placeholder": "​",
            "style": "IPY_MODEL_81b1a02bf7844ca5a6c113b608be69f1",
            "value": "Generating validation split: 100%"
          }
        },
        "6f559852c65246d095e3e215c5a9822f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_218cb93226544b42ab5b30559d7b94f5",
            "max": 3250,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d3efa9f02b9d4225a7f7e1c3a007e536",
            "value": 3250
          }
        },
        "15559040fe784a788f7b22318db88c7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_47054244877848089ba0e7a3941cdd2b",
            "placeholder": "​",
            "style": "IPY_MODEL_9d17a82715934d49b38451400545f5b4",
            "value": " 3250/3250 [00:00&lt;00:00, 49360.66 examples/s]"
          }
        },
        "4bd9daf59897421dbc159f10f7bf5443": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d28a065b2ea349ff9f7ea8424aa1f262": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "81b1a02bf7844ca5a6c113b608be69f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "218cb93226544b42ab5b30559d7b94f5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d3efa9f02b9d4225a7f7e1c3a007e536": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "47054244877848089ba0e7a3941cdd2b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9d17a82715934d49b38451400545f5b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5c53618db6d14cb1b6bb8cb2942d12c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4f70fa88519840ab950adc0dd1001c4c",
              "IPY_MODEL_6f4b6c928c1b4efc968c381e2a66debd",
              "IPY_MODEL_4688f548bad14c89a66754abbceb46ef"
            ],
            "layout": "IPY_MODEL_4acad9ad458041db9496c1e4e485ae6d"
          }
        },
        "4f70fa88519840ab950adc0dd1001c4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1623a7a53e5749788522778f9702f231",
            "placeholder": "​",
            "style": "IPY_MODEL_b954a5a576004a9c99f8806676048c58",
            "value": "Generating test split: 100%"
          }
        },
        "6f4b6c928c1b4efc968c381e2a66debd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d683c8909f394c1c963cb30f12abae4b",
            "max": 3453,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0c42261b35624129a92a2ea4cf96c31e",
            "value": 3453
          }
        },
        "4688f548bad14c89a66754abbceb46ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_43c4ed260c37423aa7791ea95bb444de",
            "placeholder": "​",
            "style": "IPY_MODEL_66551a4c938344699adb3484bf7e91bd",
            "value": " 3453/3453 [00:00&lt;00:00, 58162.28 examples/s]"
          }
        },
        "4acad9ad458041db9496c1e4e485ae6d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1623a7a53e5749788522778f9702f231": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b954a5a576004a9c99f8806676048c58": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d683c8909f394c1c963cb30f12abae4b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0c42261b35624129a92a2ea4cf96c31e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "43c4ed260c37423aa7791ea95bb444de": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "66551a4c938344699adb3484bf7e91bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b6f39c8687e24d4d845407c4b4bb5741": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5aac0c3f90bd4eaaab336d32e95841d4",
              "IPY_MODEL_8d9ba153d89f4106afae86a34caeb74c",
              "IPY_MODEL_98962f8f6e274658a8d219d198187a8c"
            ],
            "layout": "IPY_MODEL_dde04c787f4a4e8cb0d8094614a42377"
          }
        },
        "5aac0c3f90bd4eaaab336d32e95841d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b36eafd84041471ea8869f881376e4c6",
            "placeholder": "​",
            "style": "IPY_MODEL_4b688835e91e457ba0c6cb8438c37a5a",
            "value": "Map: 100%"
          }
        },
        "8d9ba153d89f4106afae86a34caeb74c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_00b273c3807a4d8587a37204b4e32cf4",
            "max": 14041,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3cbfaf2bf058480ba5ec0d3a5e6b847b",
            "value": 14041
          }
        },
        "98962f8f6e274658a8d219d198187a8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_18096f71ee544877a392b0f64ab21311",
            "placeholder": "​",
            "style": "IPY_MODEL_7b7b1d6c5ef04e82aae7beae6e77ab47",
            "value": " 14041/14041 [00:02&lt;00:00, 5271.11 examples/s]"
          }
        },
        "dde04c787f4a4e8cb0d8094614a42377": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b36eafd84041471ea8869f881376e4c6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4b688835e91e457ba0c6cb8438c37a5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "00b273c3807a4d8587a37204b4e32cf4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3cbfaf2bf058480ba5ec0d3a5e6b847b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "18096f71ee544877a392b0f64ab21311": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7b7b1d6c5ef04e82aae7beae6e77ab47": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d50e2835a3d34fb699f6a317d42d7318": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fe56ab6b6b4e49a69dbd4567f84770fb",
              "IPY_MODEL_7d0753fd498746059025f0ac57ae1b5e",
              "IPY_MODEL_17e7a48d8391420a919011b056f1b213"
            ],
            "layout": "IPY_MODEL_4f4547c92bfb44f9ae48057f06850640"
          }
        },
        "fe56ab6b6b4e49a69dbd4567f84770fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3379e0e31d8b4910a59f088055aac1fd",
            "placeholder": "​",
            "style": "IPY_MODEL_37919e0b5832450b8691c01dff07237d",
            "value": "Map: 100%"
          }
        },
        "7d0753fd498746059025f0ac57ae1b5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_50d9b1286de04449945c8975ce4deb1a",
            "max": 3250,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fa09cfcce89e44f2abde7df1fb70c593",
            "value": 3250
          }
        },
        "17e7a48d8391420a919011b056f1b213": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fafddeaf3c594e01866277b2a590f53e",
            "placeholder": "​",
            "style": "IPY_MODEL_77fac2ea13df43eba50f330d60024730",
            "value": " 3250/3250 [00:00&lt;00:00, 4105.19 examples/s]"
          }
        },
        "4f4547c92bfb44f9ae48057f06850640": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3379e0e31d8b4910a59f088055aac1fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "37919e0b5832450b8691c01dff07237d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "50d9b1286de04449945c8975ce4deb1a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fa09cfcce89e44f2abde7df1fb70c593": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fafddeaf3c594e01866277b2a590f53e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "77fac2ea13df43eba50f330d60024730": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "22aac2f873b84627b39cbed6198cc0ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_859fb6032db1426b9f58c9cca015e95b",
              "IPY_MODEL_ca23d35031f34f5fb867df55a0f61a08",
              "IPY_MODEL_05d83b29204e47d6a3ef66917cf0e076"
            ],
            "layout": "IPY_MODEL_8aa36e1cb75f49cba335620465e56a5c"
          }
        },
        "859fb6032db1426b9f58c9cca015e95b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f5a98279218c40ee92c0b084a81c43ae",
            "placeholder": "​",
            "style": "IPY_MODEL_3d67d18fe74349ceac9b09aaee132792",
            "value": "Map: 100%"
          }
        },
        "ca23d35031f34f5fb867df55a0f61a08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2102b4f2da6143d4b39bc5d82796b9c8",
            "max": 3453,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8e16a9f33bd7417e85ae46b676a61d8d",
            "value": 3453
          }
        },
        "05d83b29204e47d6a3ef66917cf0e076": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cfa7139aa2694dd4b45329d21a16de50",
            "placeholder": "​",
            "style": "IPY_MODEL_c54417dbd6ea4aff9ce083092a8e59c3",
            "value": " 3453/3453 [00:00&lt;00:00, 6335.22 examples/s]"
          }
        },
        "8aa36e1cb75f49cba335620465e56a5c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f5a98279218c40ee92c0b084a81c43ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3d67d18fe74349ceac9b09aaee132792": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2102b4f2da6143d4b39bc5d82796b9c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8e16a9f33bd7417e85ae46b676a61d8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cfa7139aa2694dd4b45329d21a16de50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c54417dbd6ea4aff9ce083092a8e59c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cb40f1c8480a460b82f0c729bfa62636": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3ab66a9356694b7082e78786e22c08a2",
              "IPY_MODEL_2e4153caa0ba49a78ca8df81133f836c",
              "IPY_MODEL_e4b09023851949398dc8e92088150a54"
            ],
            "layout": "IPY_MODEL_101858b70a1846b0a7bc86f2cdf3a369"
          }
        },
        "3ab66a9356694b7082e78786e22c08a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1b69dad583564551b309d64ef33683f8",
            "placeholder": "​",
            "style": "IPY_MODEL_a513f1d3e2e84cb099ff05510602bee2",
            "value": "Map (num_proc=4): 100%"
          }
        },
        "2e4153caa0ba49a78ca8df81133f836c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_59293fd255644a98a58d2212c182adfe",
            "max": 203621,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7c9fbbb4f2c344db9b632016e9ca970d",
            "value": 203621
          }
        },
        "e4b09023851949398dc8e92088150a54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b257eadf7fe748908231ae49fd18f084",
            "placeholder": "​",
            "style": "IPY_MODEL_cfe6f0ed2466414fbc0cff5241e86a3d",
            "value": " 203621/203621 [01:29&lt;00:00, 1186.03 examples/s]"
          }
        },
        "101858b70a1846b0a7bc86f2cdf3a369": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1b69dad583564551b309d64ef33683f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a513f1d3e2e84cb099ff05510602bee2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "59293fd255644a98a58d2212c182adfe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7c9fbbb4f2c344db9b632016e9ca970d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b257eadf7fe748908231ae49fd18f084": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cfe6f0ed2466414fbc0cff5241e86a3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e0cb0280423445fea1451c1ad172563a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e7ddc30ab7144cea86703de76a657c44",
              "IPY_MODEL_772a296129b744f1a900c1c131ec66f1",
              "IPY_MODEL_489a4337053342d09e1925547d0999f7"
            ],
            "layout": "IPY_MODEL_089c3e6b5ef043fc991627fb0c5ec04f"
          }
        },
        "e7ddc30ab7144cea86703de76a657c44": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_77bd75f759f248059864f8654a55f596",
            "placeholder": "​",
            "style": "IPY_MODEL_ca00217cdbeb46f1b284bdfa4b50d6e1",
            "value": "Map (num_proc=4): 100%"
          }
        },
        "772a296129b744f1a900c1c131ec66f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4d0483edcedf4537bbfb5078b4b9874d",
            "max": 51362,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_701d1e8442064658b697aeec55d52166",
            "value": 51362
          }
        },
        "489a4337053342d09e1925547d0999f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d693f0ba11944e7f8c5bc1a7159c85f9",
            "placeholder": "​",
            "style": "IPY_MODEL_b1bf140bbccc4efda24eee5659e7a4fc",
            "value": " 51362/51362 [00:21&lt;00:00, 2085.16 examples/s]"
          }
        },
        "089c3e6b5ef043fc991627fb0c5ec04f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "77bd75f759f248059864f8654a55f596": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ca00217cdbeb46f1b284bdfa4b50d6e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4d0483edcedf4537bbfb5078b4b9874d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "701d1e8442064658b697aeec55d52166": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d693f0ba11944e7f8c5bc1a7159c85f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b1bf140bbccc4efda24eee5659e7a4fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "15481fcdf5f24cc8ac3b26807d0698bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a2b596e1262c4af2aa50a3024e5ec655",
              "IPY_MODEL_24edbfa9a8d74524b0998639701e0a1c",
              "IPY_MODEL_7300ea5c95714230983f15d8fded772f"
            ],
            "layout": "IPY_MODEL_1fce174e4e9a4e2b83ef47a69767f8de"
          }
        },
        "a2b596e1262c4af2aa50a3024e5ec655": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c59914edc17f49d58ad34f0697c3fcbc",
            "placeholder": "​",
            "style": "IPY_MODEL_0655459d09d34526b0c5941491b88750",
            "value": "Map (num_proc=4): 100%"
          }
        },
        "24edbfa9a8d74524b0998639701e0a1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_254ce66b63be426e8c92d84d13410cc7",
            "max": 46435,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5668ba42ebc74334912851ad1e81c652",
            "value": 46435
          }
        },
        "7300ea5c95714230983f15d8fded772f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3285423ab5e8494ea8255de1ed974c5d",
            "placeholder": "​",
            "style": "IPY_MODEL_4e0856c7f0384a3fa9de3b17bafc761f",
            "value": " 46435/46435 [00:18&lt;00:00, 2564.89 examples/s]"
          }
        },
        "1fce174e4e9a4e2b83ef47a69767f8de": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c59914edc17f49d58ad34f0697c3fcbc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0655459d09d34526b0c5941491b88750": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "254ce66b63be426e8c92d84d13410cc7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5668ba42ebc74334912851ad1e81c652": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3285423ab5e8494ea8255de1ed974c5d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4e0856c7f0384a3fa9de3b17bafc761f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c573453f74eb408999aed626e6a34e7e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ae9c1133006546579a92210597eb8a60",
              "IPY_MODEL_dbccc8a6cef746e08bde588a4ca26ad9",
              "IPY_MODEL_970ee40e037a4611a8415c09ba143379"
            ],
            "layout": "IPY_MODEL_277b1871c444496d8f98c595e70bd439"
          }
        },
        "ae9c1133006546579a92210597eb8a60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0f33be08f6084dc7bc2fa3fcdde5e0cb",
            "placeholder": "​",
            "style": "IPY_MODEL_59b65bf171054beebfea59c63587a005",
            "value": "Downloading builder script: 100%"
          }
        },
        "dbccc8a6cef746e08bde588a4ca26ad9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8104fefb54a44c6b8f43a3ad07eab6b1",
            "max": 4203,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ae4f5b7dba2e448fab44fb5ddd5462dc",
            "value": 4203
          }
        },
        "970ee40e037a4611a8415c09ba143379": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d049c7a037d549fe8fbedccd46873129",
            "placeholder": "​",
            "style": "IPY_MODEL_96bbece15f4b4b64b5ce2b7fdba8ac50",
            "value": " 4.20k/4.20k [00:00&lt;00:00, 205kB/s]"
          }
        },
        "277b1871c444496d8f98c595e70bd439": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0f33be08f6084dc7bc2fa3fcdde5e0cb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "59b65bf171054beebfea59c63587a005": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8104fefb54a44c6b8f43a3ad07eab6b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ae4f5b7dba2e448fab44fb5ddd5462dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d049c7a037d549fe8fbedccd46873129": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "96bbece15f4b4b64b5ce2b7fdba8ac50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    },
    "gpuClass": "standard",
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TurkuNLP/intro-to-nlp/blob/master/sequence_labeling_mlp.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Sequence labeling (POS tagging) with MLP\n",
        "\n",
        "This notebook builds upon the [classification with MLP notebook](https://github.com/TurkuNLP/intro-to-nlp/blob/master/mlp_imdb_hf_dset_and_trainer.ipynb) and shows how to implement a basic sequence labeling method."
      ],
      "metadata": {
        "id": "arPRayYSpzJs"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Setup\n",
        "\n",
        "Install the required Python packages using [pip](https://en.wikipedia.org/wiki/Pip):\n",
        "\n",
        "* [`transformers`](https://huggingface.co/docs/transformers/index) is a popular deep learning package primarily on top of torch\n",
        "* [`datasets`](https://huggingface.co/docs/datasets/) provides support for loading, creating, and manipulating datasets\n",
        "* [`evaluate`](https://huggingface.co/docs/evaluate/index) is a library of performance metrics (like accuracy etc)"
      ],
      "metadata": {
        "id": "LDZZUKzfPRWT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --quiet transformers[torch] datasets evaluate"
      ],
      "metadata": {
        "id": "fKPHBYptQDsK",
        "outputId": "90f84b29-9c0f-41d7-b526-b89002921f3d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/510.5 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.4/510.5 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m \u001b[32m501.8/510.5 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m510.5/510.5 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m290.1/290.1 kB\u001b[0m \u001b[31m26.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m12.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m12.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m33.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m46.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m71.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m15.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m6.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m13.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.1/21.1 MB\u001b[0m \u001b[31m56.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Get and prepare data\n",
        "\n",
        "*   Let us work with the venerable, if somewhat dated [CoNLL'03 shared task](https://aclanthology.org/W03-0419.pdf) English data\n",
        "*   These are English news articles, and have annotation for POS, syntactic chunks, and named entities (in the IOB format)\n",
        "\n",
        "The data as originally distributed for the 2003 shared task has the following format:\n",
        "\n",
        "```\n",
        "Only RB B-NP O\n",
        "France NNP I-NP B-LOC\n",
        "and CC I-NP O\n",
        "Britain NNP I-NP B-LOC\n",
        "backed VBD B-VP O\n",
        "Fischler NNP B-NP B-PER\n",
        "'s POS I-NP O\n",
        "proposal NN I-NP O\n",
        ". . O O\n",
        "```\n",
        "\n",
        "Here, the four space-separated columns are token text, POS tag, chunk tag, and NER tag. The goal of the original task is to predict the NER tags using the other information as features, but the dataset can be used to study predicting the other columns too.\n",
        "\n",
        "The dataset happens to be in the HF datasets collection, so we can grab it from there\n"
      ],
      "metadata": {
        "id": "_fCdfQfNNzwt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import transformers\n",
        "import datasets\n",
        "\n",
        "from pprint import pprint    # pretty-print\n",
        "\n",
        "dataset = datasets.load_dataset(\"conll2003\")\n",
        "\n",
        "print(dataset)"
      ],
      "metadata": {
        "id": "QxmgHoKDTN2_",
        "outputId": "8b01909a-2df0-4ffb-c205-db99b812c93f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 576,
          "referenced_widgets": [
            "3b7e4bb9a3334cf3b0d7a09629c5d703",
            "d641e90a212744199f7d98af7791e79e",
            "fed0221314094363a37114517e917dcf",
            "18e0eb340f5d4a8da8fad50c21f8161e",
            "198f8ea06b2a4fae89fc5449c1fc61a6",
            "21a3f45ce81f4430bfa0dfda3969d9d0",
            "8db7b271b6024c5dbf9c17ec222dd93a",
            "5bb9cded71ae4cc5901e27fe1f9b7566",
            "622e5f4f4ed9471fa78a6652eb1b9812",
            "874542c532694e1fa4cc46b91f9edb48",
            "19fc9de7cfb9483b85b20f11b8871f31",
            "fd65ed9d7f8e43c083334c70096ab97f",
            "69c38d47a26d45cfb977e77f963d571e",
            "7ec7d73c10ea4605b756cdc888987a8d",
            "654e236e93cd467a98e28a2aca9f2feb",
            "60690be8d9ee41769a5cbdb7d6950c92",
            "28e4758925884be698ee23806cdda147",
            "291dc23cd40f4f6094c25854f8851bbd",
            "bb3c99c42fd14e339e7c07b9b1a88e25",
            "f1f90b6362c54e9f835c40ff50c54fef",
            "9d1d0c33523945d69667e45dbb67d536",
            "6c202e42e2414cd4a5f871466cb1391d",
            "773d0c2f62c74a3296538a664e507b3b",
            "43882e01df1c41469fed318c2b525ada",
            "e6013f778a6e4851ae56acffbd4e0d74",
            "ca7893865fff415f96b042c6cbd5904c",
            "e85560080b6c47d7a6169dfd76389ecf",
            "436144cba76d4eb69d2f97d8cd9d49c1",
            "a58c9522f7584b3ca1c9b053596a1646",
            "5558da787ca240d4a90cc924f15b9f8b",
            "a8530e4977094ee2b8ba29de3778e6ce",
            "181e26da59174240a9999520ee035a5c",
            "16acb247a1ff4fd0b362cfd08b0f23ec",
            "10d247583d8b45188a8cf1c8554a4e33",
            "123fe0647fd84174ae030017d4dcbf6c",
            "6aa220e9a81d4d1296de9f91926246b4",
            "27d6857a0f5b4301a9c4fec5325960d2",
            "da99f48993ee4c168b2076ae7a22861f",
            "9e351ef8fd2b4710ad73ba9aa2fbc4a2",
            "e07368c90ef548aa8e54e3106b9df600",
            "c5c91ac98d244093a4a7c20ae82d2759",
            "ca1e23759298416b939ebb8b3e1d5538",
            "16ce1d1f13ce4dfa9b3279cfb9e4623a",
            "51d00181226b4b949519f76583c1dd78",
            "d94f7c6b4c874299854dfa90495ad550",
            "b43511af23c2499bb841866a5a68f1ef",
            "6f559852c65246d095e3e215c5a9822f",
            "15559040fe784a788f7b22318db88c7f",
            "4bd9daf59897421dbc159f10f7bf5443",
            "d28a065b2ea349ff9f7ea8424aa1f262",
            "81b1a02bf7844ca5a6c113b608be69f1",
            "218cb93226544b42ab5b30559d7b94f5",
            "d3efa9f02b9d4225a7f7e1c3a007e536",
            "47054244877848089ba0e7a3941cdd2b",
            "9d17a82715934d49b38451400545f5b4",
            "5c53618db6d14cb1b6bb8cb2942d12c7",
            "4f70fa88519840ab950adc0dd1001c4c",
            "6f4b6c928c1b4efc968c381e2a66debd",
            "4688f548bad14c89a66754abbceb46ef",
            "4acad9ad458041db9496c1e4e485ae6d",
            "1623a7a53e5749788522778f9702f231",
            "b954a5a576004a9c99f8806676048c58",
            "d683c8909f394c1c963cb30f12abae4b",
            "0c42261b35624129a92a2ea4cf96c31e",
            "43c4ed260c37423aa7791ea95bb444de",
            "66551a4c938344699adb3484bf7e91bd"
          ]
        }
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/1.23M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3b7e4bb9a3334cf3b0d7a09629c5d703"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/312k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fd65ed9d7f8e43c083334c70096ab97f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/283k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "773d0c2f62c74a3296538a664e507b3b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/14041 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "10d247583d8b45188a8cf1c8554a4e33"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating validation split:   0%|          | 0/3250 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d94f7c6b4c874299854dfa90495ad550"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating test split:   0%|          | 0/3453 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5c53618db6d14cb1b6bb8cb2942d12c7"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "DatasetDict({\n",
            "    train: Dataset({\n",
            "        features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags'],\n",
            "        num_rows: 14041\n",
            "    })\n",
            "    validation: Dataset({\n",
            "        features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags'],\n",
            "        num_rows: 3250\n",
            "    })\n",
            "    test: Dataset({\n",
            "        features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags'],\n",
            "        num_rows: 3453\n",
            "    })\n",
            "})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pprint(dataset[\"train\"][12])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OEtbYIyLg3r7",
        "outputId": "266c123b-cfc5-41aa-e281-9acf994dd1cc"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'chunk_tags': [11, 12, 12, 12, 21, 11, 11, 12, 0],\n",
            " 'id': '12',\n",
            " 'ner_tags': [0, 5, 0, 5, 0, 1, 0, 0, 0],\n",
            " 'pos_tags': [30, 22, 10, 22, 38, 22, 27, 21, 7],\n",
            " 'tokens': ['Only',\n",
            "            'France',\n",
            "            'and',\n",
            "            'Britain',\n",
            "            'backed',\n",
            "            'Fischler',\n",
            "            \"'s\",\n",
            "            'proposal',\n",
            "            '.']}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "As you can see above, the various labels (POS, NER and chunk tags) are converted into IDs in this dataset. We can access the textual labels of these tags through the dataset `features`:"
      ],
      "metadata": {
        "id": "v9TMq-d3qFEJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "POS_TAG_NAMES = dataset['train'].features['pos_tags'].feature.names\n",
        "NER_TAG_NAMES = dataset['train'].features['ner_tags'].feature.names\n",
        "CHUNK_TAG_NAMES = dataset['train'].features['chunk_tags'].feature.names"
      ],
      "metadata": {
        "id": "u0kflH3RpHKY"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can then create mappings from names to IDs and back as Python dictionaries:"
      ],
      "metadata": {
        "id": "O3-QCWMvrazc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "POS2ID = { n: i for i, n in enumerate(POS_TAG_NAMES) }\n",
        "ID2POS = { i: n for i, n in enumerate(POS_TAG_NAMES) }\n",
        "\n",
        "NER2ID = { n: i for i, n in enumerate(NER_TAG_NAMES) }\n",
        "ID2NER = { i: n for i, n in enumerate(NER_TAG_NAMES) }\n",
        "\n",
        "CHUNK2ID = { n: i for i, n in enumerate(CHUNK_TAG_NAMES) }\n",
        "ID2CHUNK = { i: n for i, n in enumerate(CHUNK_TAG_NAMES) }"
      ],
      "metadata": {
        "id": "tqi_FN2yq7MN"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "This is what these mappings look like:"
      ],
      "metadata": {
        "id": "sUI-FiAssDLL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(NER2ID)"
      ],
      "metadata": {
        "id": "XAYSIMWjsFiY",
        "outputId": "2d72bdcc-6a50-45f7-d5a9-7d52e9fc2292",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'O': 0, 'B-PER': 1, 'I-PER': 2, 'B-ORG': 3, 'I-ORG': 4, 'B-LOC': 5, 'I-LOC': 6, 'B-MISC': 7, 'I-MISC': 8}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(ID2NER)"
      ],
      "metadata": {
        "id": "NO4xI0yasHdM",
        "outputId": "689b01fe-87fd-4537-cbef-344d82be07c7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{0: 'O', 1: 'B-PER', 2: 'I-PER', 3: 'B-ORG', 4: 'I-ORG', 5: 'B-LOC', 6: 'I-LOC', 7: 'B-MISC', 8: 'I-MISC'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's also add in explanations from Penn Treebank for the POS tags:"
      ],
      "metadata": {
        "id": "jUqKBvtpvt3G"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# From the documentation page and from here https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html\n",
        "\n",
        "POS2DESCRIPTION = {\n",
        "    \"CC\": \"Coordinating conjunction\",\n",
        "    \"CD\": \"Cardinal number\",\n",
        "    \"DT\": \"Determiner\",\n",
        "    \"EX\": \"Existential there\",\n",
        "    \"FW\": \"Foreign word\",\n",
        "    \"IN\": \"Preposition or subordinating conjunction\",\n",
        "    \"JJ\": \"Adjective\",\n",
        "    \"JJR\": \"Adjective, comparative\",\n",
        "    \"JJS\": \"Adjective, superlative\",\n",
        "    \"LS\": \"List item marker\",\n",
        "    \"MD\": \"Modal\",\n",
        "    \"NN\": \"Noun, singular or mass\",\n",
        "    \"NNS\": \"Noun, plural\",\n",
        "    \"NNP\": \"Proper noun, singular\",\n",
        "    \"NNPS\": \"Proper noun, plural\",\n",
        "    \"PDT\": \"Predeterminer\",\n",
        "    \"POS\": \"Possessive ending\",\n",
        "    \"PRP\": \"Personal pronoun\",\n",
        "    \"PRP$\": \"Possessive pronoun\",\n",
        "    \"RB\": \"Adverb\",\n",
        "    \"RBR\": \"Adverb, comparative\",\n",
        "    \"RBS\": \"Adverb, superlative\",\n",
        "    \"RP\": \"Particle\",\n",
        "    \"SYM\": \"Symbol\",\n",
        "    \"TO\": \"to\",\n",
        "    \"UH\": \"Interjection\",\n",
        "    \"VB\": \"Verb, base form\",\n",
        "    \"VBD\": \"Verb, past tense\",\n",
        "    \"VBG\": \"Verb, gerund or present participle\",\n",
        "    \"VBN\": \"Verb, past participle\",\n",
        "    \"VBP\": \"Verb, non-3rd person singular present\",\n",
        "    \"VBZ\": \"Verb, 3rd person singular present\",\n",
        "    \"WDT\": \"Wh-determiner\",\n",
        "    \"WP\": \"Wh-pronoun\",\n",
        "    \"WP$\": \"Possessive wh-pronoun\",\n",
        "    \"WRB\": \"Wh-adverb\"\n",
        "}"
      ],
      "metadata": {
        "id": "EDIG9IkNf_ji"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can now try to make sense of the tags:"
      ],
      "metadata": {
        "id": "HRcNtMskyTxq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import tabulate\n",
        "\n",
        "e = dataset[\"train\"][12]    # work on the same example\n",
        "\n",
        "table = []\n",
        "for token, pos_id, chunk_id, ner_id in zip(e[\"tokens\"], e[\"pos_tags\"], e[\"chunk_tags\"], e[\"ner_tags\"]):\n",
        "    ner_tag = ID2NER[ner_id]\n",
        "    chunk_tag = ID2CHUNK[chunk_id]\n",
        "    pos_tag = ID2POS[pos_id]\n",
        "    pos_def = POS2DESCRIPTION.get(pos_tag,pos_tag)\n",
        "    table.append([token, ner_tag, chunk_tag, pos_tag, pos_def])\n",
        "\n",
        "print(tabulate.tabulate(table,headers=[\"Token\", \"NER\", \"Chunk\", \"POS\", \"POS definition\"]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JzQXQI0YiD0_",
        "outputId": "b8b23c5e-52e2-465d-d5f7-747c3fc9fb90"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Token     NER    Chunk    POS    POS definition\n",
            "--------  -----  -------  -----  ------------------------\n",
            "Only      O      B-NP     RB     Adverb\n",
            "France    B-LOC  I-NP     NNP    Proper noun, singular\n",
            "and       O      I-NP     CC     Coordinating conjunction\n",
            "Britain   B-LOC  I-NP     NNP    Proper noun, singular\n",
            "backed    O      B-VP     VBD    Verb, past tense\n",
            "Fischler  B-PER  B-NP     NNP    Proper noun, singular\n",
            "'s        O      B-NP     POS    Possessive ending\n",
            "proposal  O      I-NP     NN     Noun, singular or mass\n",
            ".         O      O        .      .\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Note that the data is organized into sentences."
      ],
      "metadata": {
        "id": "VXm-E32AdVpo"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Create features\n",
        "\n",
        "We'll define a simple function that takes a token sequence, the index of the focus token, and a window size and generates a few basic explicit features relevant to the task.\n",
        "\n",
        "(Note that as we'll be predicting the POS tag, we won't look at the chunk or NER tags, which would typically only be predicted _after_ predicting POS in a \"traditional\" NLP pipeline)"
      ],
      "metadata": {
        "id": "d9WvQiwluhBO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def token_features(tokens, index, window_size):\n",
        "    # Generate features for token in position `index` in given list of tokens\n",
        "    features = []\n",
        "\n",
        "    # Context window start and end\n",
        "    window_start = max(0, index-window_size)\n",
        "    window_end = min(index+window_size+1, len(tokens))    # note +1 for range\n",
        "\n",
        "    for i in range(window_start, window_end):\n",
        "          offset = i - index    # relative position\n",
        "          features.append(f\"token[{offset}]={tokens[i]}\")\n",
        "\n",
        "    # Example custom feature: does focus token start with an upper-case letter?\n",
        "    if tokens[index][0].isupper():\n",
        "        features.append(\"first-letter-capitalized\")\n",
        "\n",
        "    return features"
      ],
      "metadata": {
        "id": "6NjCVYHWe32j"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can call this function for all tokens in a sentence like so:"
      ],
      "metadata": {
        "id": "k1aEFA2Yg8xO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def add_features_to_sentence(sentence):\n",
        "    # Collect lists of features for all tokens here\n",
        "    all_features = []\n",
        "\n",
        "    tokens = sentence[\"tokens\"]\n",
        "    for index in range(len(tokens)):\n",
        "        all_features.append(token_features(tokens, index, window_size=3))\n",
        "\n",
        "    return { \"features\": all_features }"
      ],
      "metadata": {
        "id": "tgTZ5h4hhGFd"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for feats in add_features_to_sentence(dataset[\"train\"][12])[\"features\"]:\n",
        "    print(feats)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VMoD24fQhnRK",
        "outputId": "b449b9ab-8c8c-4ce9-afa0-4e5f4732f36c"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['token[0]=Only', 'token[1]=France', 'token[2]=and', 'token[3]=Britain', 'first-letter-capitalized']\n",
            "['token[-1]=Only', 'token[0]=France', 'token[1]=and', 'token[2]=Britain', 'token[3]=backed', 'first-letter-capitalized']\n",
            "['token[-2]=Only', 'token[-1]=France', 'token[0]=and', 'token[1]=Britain', 'token[2]=backed', 'token[3]=Fischler']\n",
            "['token[-3]=Only', 'token[-2]=France', 'token[-1]=and', 'token[0]=Britain', 'token[1]=backed', 'token[2]=Fischler', \"token[3]='s\", 'first-letter-capitalized']\n",
            "['token[-3]=France', 'token[-2]=and', 'token[-1]=Britain', 'token[0]=backed', 'token[1]=Fischler', \"token[2]='s\", 'token[3]=proposal']\n",
            "['token[-3]=and', 'token[-2]=Britain', 'token[-1]=backed', 'token[0]=Fischler', \"token[1]='s\", 'token[2]=proposal', 'token[3]=.', 'first-letter-capitalized']\n",
            "['token[-3]=Britain', 'token[-2]=backed', 'token[-1]=Fischler', \"token[0]='s\", 'token[1]=proposal', 'token[2]=.']\n",
            "['token[-3]=backed', 'token[-2]=Fischler', \"token[-1]='s\", 'token[0]=proposal', 'token[1]=.']\n",
            "['token[-3]=Fischler', \"token[-2]='s\", 'token[-1]=proposal', 'token[0]=.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The dataset is organized into sentences, so we can use the above function to add features to the entire dataset as follows.\n",
        "\n",
        "**Note**: unlike e.g. the Python`map` function, [`Dataset.map`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map) function _updates_ its argument dataset, keeping existing values."
      ],
      "metadata": {
        "id": "c3SlJhk7i8aV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = dataset.map(add_features_to_sentence)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "b6f39c8687e24d4d845407c4b4bb5741",
            "5aac0c3f90bd4eaaab336d32e95841d4",
            "8d9ba153d89f4106afae86a34caeb74c",
            "98962f8f6e274658a8d219d198187a8c",
            "dde04c787f4a4e8cb0d8094614a42377",
            "b36eafd84041471ea8869f881376e4c6",
            "4b688835e91e457ba0c6cb8438c37a5a",
            "00b273c3807a4d8587a37204b4e32cf4",
            "3cbfaf2bf058480ba5ec0d3a5e6b847b",
            "18096f71ee544877a392b0f64ab21311",
            "7b7b1d6c5ef04e82aae7beae6e77ab47",
            "d50e2835a3d34fb699f6a317d42d7318",
            "fe56ab6b6b4e49a69dbd4567f84770fb",
            "7d0753fd498746059025f0ac57ae1b5e",
            "17e7a48d8391420a919011b056f1b213",
            "4f4547c92bfb44f9ae48057f06850640",
            "3379e0e31d8b4910a59f088055aac1fd",
            "37919e0b5832450b8691c01dff07237d",
            "50d9b1286de04449945c8975ce4deb1a",
            "fa09cfcce89e44f2abde7df1fb70c593",
            "fafddeaf3c594e01866277b2a590f53e",
            "77fac2ea13df43eba50f330d60024730",
            "22aac2f873b84627b39cbed6198cc0ac",
            "859fb6032db1426b9f58c9cca015e95b",
            "ca23d35031f34f5fb867df55a0f61a08",
            "05d83b29204e47d6a3ef66917cf0e076",
            "8aa36e1cb75f49cba335620465e56a5c",
            "f5a98279218c40ee92c0b084a81c43ae",
            "3d67d18fe74349ceac9b09aaee132792",
            "2102b4f2da6143d4b39bc5d82796b9c8",
            "8e16a9f33bd7417e85ae46b676a61d8d",
            "cfa7139aa2694dd4b45329d21a16de50",
            "c54417dbd6ea4aff9ce083092a8e59c3"
          ]
        },
        "id": "R9dekapGjNcb",
        "outputId": "57a08c4e-0ab2-4873-bf8e-8f0341dc3f3c"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/14041 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b6f39c8687e24d4d845407c4b4bb5741"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/3250 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d50e2835a3d34fb699f6a317d42d7318"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/3453 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "22aac2f873b84627b39cbed6198cc0ac"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's check that one more time:"
      ],
      "metadata": {
        "id": "Ke-vVsAulwMb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pprint(dataset[\"train\"][12])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7zfUEZbDlyeZ",
        "outputId": "b132cf83-57be-442d-d440-82312603f0d8"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'chunk_tags': [11, 12, 12, 12, 21, 11, 11, 12, 0],\n",
            " 'features': [['token[0]=Only',\n",
            "               'token[1]=France',\n",
            "               'token[2]=and',\n",
            "               'token[3]=Britain',\n",
            "               'first-letter-capitalized'],\n",
            "              ['token[-1]=Only',\n",
            "               'token[0]=France',\n",
            "               'token[1]=and',\n",
            "               'token[2]=Britain',\n",
            "               'token[3]=backed',\n",
            "               'first-letter-capitalized'],\n",
            "              ['token[-2]=Only',\n",
            "               'token[-1]=France',\n",
            "               'token[0]=and',\n",
            "               'token[1]=Britain',\n",
            "               'token[2]=backed',\n",
            "               'token[3]=Fischler'],\n",
            "              ['token[-3]=Only',\n",
            "               'token[-2]=France',\n",
            "               'token[-1]=and',\n",
            "               'token[0]=Britain',\n",
            "               'token[1]=backed',\n",
            "               'token[2]=Fischler',\n",
            "               \"token[3]='s\",\n",
            "               'first-letter-capitalized'],\n",
            "              ['token[-3]=France',\n",
            "               'token[-2]=and',\n",
            "               'token[-1]=Britain',\n",
            "               'token[0]=backed',\n",
            "               'token[1]=Fischler',\n",
            "               \"token[2]='s\",\n",
            "               'token[3]=proposal'],\n",
            "              ['token[-3]=and',\n",
            "               'token[-2]=Britain',\n",
            "               'token[-1]=backed',\n",
            "               'token[0]=Fischler',\n",
            "               \"token[1]='s\",\n",
            "               'token[2]=proposal',\n",
            "               'token[3]=.',\n",
            "               'first-letter-capitalized'],\n",
            "              ['token[-3]=Britain',\n",
            "               'token[-2]=backed',\n",
            "               'token[-1]=Fischler',\n",
            "               \"token[0]='s\",\n",
            "               'token[1]=proposal',\n",
            "               'token[2]=.'],\n",
            "              ['token[-3]=backed',\n",
            "               'token[-2]=Fischler',\n",
            "               \"token[-1]='s\",\n",
            "               'token[0]=proposal',\n",
            "               'token[1]=.'],\n",
            "              ['token[-3]=Fischler',\n",
            "               \"token[-2]='s\",\n",
            "               'token[-1]=proposal',\n",
            "               'token[0]=.']],\n",
            " 'id': '12',\n",
            " 'ner_tags': [0, 5, 0, 5, 0, 1, 0, 0, 0],\n",
            " 'pos_tags': [30, 22, 10, 22, 38, 22, 27, 21, 7],\n",
            " 'tokens': ['Only',\n",
            "            'France',\n",
            "            'and',\n",
            "            'Britain',\n",
            "            'backed',\n",
            "            'Fischler',\n",
            "            \"'s\",\n",
            "            'proposal',\n",
            "            '.']}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Flatten dataset\n",
        "\n",
        "The MLP code that we introduced previously expects each of the `train`, `validation` and `test` subsets of the data to consist of simple sequences of examples.\n",
        "\n",
        "Now that we have run the feature generation, we no longer need the sentence structure and can \"flatten\" the data into such sequences."
      ],
      "metadata": {
        "id": "N8sZzkiOl9aZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def flatten(subset):\n",
        "    # Keys for values to flatten\n",
        "    keys = [\"tokens\", \"pos_tags\", \"chunk_tags\", \"ner_tags\", \"features\"]\n",
        "\n",
        "    # Initialize to empty lists of tokens etc.\n",
        "    flattened = { k: [] for k in keys }\n",
        "\n",
        "    # Concatenate per-sentence lists of tokens etc.\n",
        "    for sentence in subset:\n",
        "        for key in keys:\n",
        "            flattened[key].extend(sentence[key])\n",
        "\n",
        "    # Return as Dataset object\n",
        "    return datasets.Dataset.from_dict(flattened)"
      ],
      "metadata": {
        "id": "Y4COBECNjPNb"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Call `flatten` for each of the subsets and make a new `DatasetDict` containing the flattened subsets:"
      ],
      "metadata": {
        "id": "wUuPc9uwltTn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "flattened_dict = {\n",
        "    \"train\": flatten(dataset[\"train\"]),\n",
        "    \"validation\": flatten(dataset[\"validation\"]),\n",
        "    \"test\": flatten(dataset[\"test\"]),\n",
        "}\n",
        "\n",
        "flat_dataset = datasets.DatasetDict(flattened_dict)"
      ],
      "metadata": {
        "id": "jcQSwn2Ll6Z0"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Check that the new dataset looks OK:"
      ],
      "metadata": {
        "id": "QR4PZp8AmWZ_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "flat_dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H9xBRcJKkB7Q",
        "outputId": "cce8013f-fbf5-42ca-c9af-70e5c398dff7"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['tokens', 'pos_tags', 'chunk_tags', 'ner_tags', 'features'],\n",
              "        num_rows: 203621\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['tokens', 'pos_tags', 'chunk_tags', 'ner_tags', 'features'],\n",
              "        num_rows: 51362\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['tokens', 'pos_tags', 'chunk_tags', 'ner_tags', 'features'],\n",
              "        num_rows: 46435\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(10):\n",
        "    token = flat_dataset[\"train\"][\"tokens\"][i]\n",
        "    pos_tag = ID2POS[flat_dataset[\"train\"][\"pos_tags\"][i]]\n",
        "    description = POS2DESCRIPTION.get(pos_tag, pos_tag)\n",
        "    features = flat_dataset[\"train\"][\"features\"][i]\n",
        "    print(f\"{token}\\t{pos_tag}\\t{description}\\t{features}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vs55SS3bmrqz",
        "outputId": "022818aa-f848-44c3-f4d5-79d427a6f642"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "EU\tNNP\tProper noun, singular\t['token[0]=EU', 'token[1]=rejects', 'token[2]=German', 'token[3]=call', 'first-letter-capitalized']\n",
            "rejects\tVBZ\tVerb, 3rd person singular present\t['token[-1]=EU', 'token[0]=rejects', 'token[1]=German', 'token[2]=call', 'token[3]=to']\n",
            "German\tJJ\tAdjective\t['token[-2]=EU', 'token[-1]=rejects', 'token[0]=German', 'token[1]=call', 'token[2]=to', 'token[3]=boycott', 'first-letter-capitalized']\n",
            "call\tNN\tNoun, singular or mass\t['token[-3]=EU', 'token[-2]=rejects', 'token[-1]=German', 'token[0]=call', 'token[1]=to', 'token[2]=boycott', 'token[3]=British']\n",
            "to\tTO\tto\t['token[-3]=rejects', 'token[-2]=German', 'token[-1]=call', 'token[0]=to', 'token[1]=boycott', 'token[2]=British', 'token[3]=lamb']\n",
            "boycott\tVB\tVerb, base form\t['token[-3]=German', 'token[-2]=call', 'token[-1]=to', 'token[0]=boycott', 'token[1]=British', 'token[2]=lamb', 'token[3]=.']\n",
            "British\tJJ\tAdjective\t['token[-3]=call', 'token[-2]=to', 'token[-1]=boycott', 'token[0]=British', 'token[1]=lamb', 'token[2]=.', 'first-letter-capitalized']\n",
            "lamb\tNN\tNoun, singular or mass\t['token[-3]=to', 'token[-2]=boycott', 'token[-1]=British', 'token[0]=lamb', 'token[1]=.']\n",
            ".\t.\t.\t['token[-3]=boycott', 'token[-2]=British', 'token[-1]=lamb', 'token[0]=.']\n",
            "Peter\tNNP\tProper noun, singular\t['token[0]=Peter', 'token[1]=Blackburn', 'first-letter-capitalized']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Note that this is now a single long sequence of tokens without sentence boundaries."
      ],
      "metadata": {
        "id": "lI0nCoZhnode"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "## Vectorize data\n",
        "\n",
        "We'll next follow the steps that you should already be familiar with from the [text classification notebook](https://github.com/TurkuNLP/intro-to-nlp/blob/master/mlp_imdb_hf_dset_and_trainer.ipynb), with a few changes:\n",
        "\n",
        "* Since the data is already tokenized, we only need to **vectorize** it, i.e. get the non-zero elements of the feature vector\n",
        "* Unlike in the text classification notebook, here we are **vectorizing token features**\n",
        "* We'll again use sklearn's feature extraction package, in particular `CountVectorizer`\n",
        "* Since our features are now lists of strings, we can skip tokenization and use these as-is"
      ],
      "metadata": {
        "id": "KMnqQ78cMpk2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import sklearn.feature_extraction\n",
        "\n",
        "\n",
        "# Dummy function for tokenization and preprocessing\n",
        "def do_nothing(features):\n",
        "    return features\n",
        "\n",
        "vectorizer = sklearn.feature_extraction.text.CountVectorizer(\n",
        "    binary=True,\n",
        "    max_features=30000,\n",
        "    tokenizer=do_nothing,\n",
        "    preprocessor=do_nothing,\n",
        ")\n",
        "\n",
        "# Get a list of all feature strings from the training data\n",
        "features = [e[\"features\"] for e in flat_dataset[\"train\"]]\n",
        "\n",
        "# \"Train\" the vectorizer, i.e. build its vocabulary\n",
        "vectorizer.fit(features)"
      ],
      "metadata": {
        "id": "MYOtKJu7Mohd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 109
        },
        "outputId": "f1ef0788-c80e-497b-9520-c6551c45723a"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "CountVectorizer(binary=True, max_features=30000,\n",
              "                preprocessor=<function do_nothing at 0x78567c2ed750>,\n",
              "                tokenizer=<function do_nothing at 0x78567c2ed750>)"
            ],
            "text/html": [
              "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>CountVectorizer(binary=True, max_features=30000,\n",
              "                preprocessor=&lt;function do_nothing at 0x78567c2ed750&gt;,\n",
              "                tokenizer=&lt;function do_nothing at 0x78567c2ed750&gt;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">CountVectorizer</label><div class=\"sk-toggleable__content\"><pre>CountVectorizer(binary=True, max_features=30000,\n",
              "                preprocessor=&lt;function do_nothing at 0x78567c2ed750&gt;,\n",
              "                tokenizer=&lt;function do_nothing at 0x78567c2ed750&gt;)</pre></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "As in the text classification notebook, we then invoke the vectorizer and get non-zero elements as a sparse matrix:"
      ],
      "metadata": {
        "id": "UDNy3w8reO6C"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def vectorize_example(e):\n",
        "    vectorized = vectorizer.transform([e[\"features\"]])\n",
        "\n",
        "    # nonzero() gives a pair of (rows,columns), we want the columns\n",
        "    non_zero_features = vectorized.nonzero()[1]\n",
        "\n",
        "    # Feature index 0 will have a special meaning, so let us not produce\n",
        "    # it by adding +1 to everything\n",
        "    non_zero_features += 1\n",
        "\n",
        "    return {\n",
        "        \"input_ids\": non_zero_features,\n",
        "        \"label\": e[\"pos_tags\"]\n",
        "    }"
      ],
      "metadata": {
        "id": "5H9AQOc2rGIO"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Check one example:"
      ],
      "metadata": {
        "id": "kqKjaSmBqWjm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vectorized = vectorize_example(flat_dataset[\"train\"][10])\n",
        "\n",
        "print(flat_dataset[\"train\"][10])\n",
        "print(vectorized)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EOTjsFCBqOjX",
        "outputId": "a39914ee-475b-4969-8dd6-678d27c26c55"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'tokens': 'Blackburn', 'pos_tags': 22, 'chunk_tags': 12, 'ner_tags': 2, 'features': ['token[-1]=Peter', 'token[0]=Blackburn', 'first-letter-capitalized']}\n",
            "{'input_ids': array([    1,  1603, 13770], dtype=int32), 'label': 22}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Map `input_ids` back to the original feature names to confirm that everything works:"
      ],
      "metadata": {
        "id": "WiScO867qrjK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Invert the feature dictionary\n",
        "idx2feat = { i: w for w, i in vectorizer.vocabulary_.items() }\n",
        "\n",
        "feats = []\n",
        "for idx in vectorized[\"input_ids\"]:\n",
        "    feats.append(idx2feat[idx-1])    # It is easy to forget we moved all by +1\n",
        "\n",
        "# This is now the bag of features representation of the token in context\n",
        "pprint(\", \".join(feats))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yvgd3IrWfCai",
        "outputId": "68d2c5e9-d073-4c2f-e972-d4ead8250683"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "'first-letter-capitalized, token[-1]=Peter, token[0]=Blackburn'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Vectorizing the whole dataset\n",
        "\n",
        "We'll again use [`Dataset.map`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map) to process the whole dataset:"
      ],
      "metadata": {
        "id": "i33KIMgkiIn7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vectorized_dataset = flat_dataset.map(vectorize_example, num_proc=4)\n",
        "\n",
        "pprint(vectorized_dataset[\"train\"][0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 304,
          "referenced_widgets": [
            "cb40f1c8480a460b82f0c729bfa62636",
            "3ab66a9356694b7082e78786e22c08a2",
            "2e4153caa0ba49a78ca8df81133f836c",
            "e4b09023851949398dc8e92088150a54",
            "101858b70a1846b0a7bc86f2cdf3a369",
            "1b69dad583564551b309d64ef33683f8",
            "a513f1d3e2e84cb099ff05510602bee2",
            "59293fd255644a98a58d2212c182adfe",
            "7c9fbbb4f2c344db9b632016e9ca970d",
            "b257eadf7fe748908231ae49fd18f084",
            "cfe6f0ed2466414fbc0cff5241e86a3d",
            "e0cb0280423445fea1451c1ad172563a",
            "e7ddc30ab7144cea86703de76a657c44",
            "772a296129b744f1a900c1c131ec66f1",
            "489a4337053342d09e1925547d0999f7",
            "089c3e6b5ef043fc991627fb0c5ec04f",
            "77bd75f759f248059864f8654a55f596",
            "ca00217cdbeb46f1b284bdfa4b50d6e1",
            "4d0483edcedf4537bbfb5078b4b9874d",
            "701d1e8442064658b697aeec55d52166",
            "d693f0ba11944e7f8c5bc1a7159c85f9",
            "b1bf140bbccc4efda24eee5659e7a4fc",
            "15481fcdf5f24cc8ac3b26807d0698bb",
            "a2b596e1262c4af2aa50a3024e5ec655",
            "24edbfa9a8d74524b0998639701e0a1c",
            "7300ea5c95714230983f15d8fded772f",
            "1fce174e4e9a4e2b83ef47a69767f8de",
            "c59914edc17f49d58ad34f0697c3fcbc",
            "0655459d09d34526b0c5941491b88750",
            "254ce66b63be426e8c92d84d13410cc7",
            "5668ba42ebc74334912851ad1e81c652",
            "3285423ab5e8494ea8255de1ed974c5d",
            "4e0856c7f0384a3fa9de3b17bafc761f"
          ]
        },
        "id": "33xMYRd0q2B9",
        "outputId": "397cab16-b1df-4283-e32c-7c435377b4d0"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/203621 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cb40f1c8480a460b82f0c729bfa62636"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/51362 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e0cb0280423445fea1451c1ad172563a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/46435 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "15481fcdf5f24cc8ac3b26807d0698bb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'chunk_tags': 11,\n",
            " 'features': ['token[0]=EU',\n",
            "              'token[1]=rejects',\n",
            "              'token[2]=German',\n",
            "              'token[3]=call',\n",
            "              'first-letter-capitalized'],\n",
            " 'input_ids': [1, 14072, 23002, 27892],\n",
            " 'label': 22,\n",
            " 'ner_tags': 3,\n",
            " 'pos_tags': 22,\n",
            " 'tokens': 'EU'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "* Our `input_ids` are an array containing the indices of the features\n",
        "* This corresponds to the indices into the row of the embedding matrix in the model\n"
      ],
      "metadata": {
        "id": "XTtyHQpIIJWw"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Batching and padding\n",
        "\n",
        "As detailed in the [text classification notebook](https://github.com/TurkuNLP/intro-to-nlp/blob/master/mlp_imdb_hf_dset_and_trainer.ipynb), we typically train neural networks on _batches_ of multiple examples rather than a single example at a time (efficiency and regularization).\n",
        "\n",
        "As examples in a batch need to have identical length, we _pad_ shorter examples to the maximum example length in each batch with the \"dummy\" feature with index 0.\n",
        "\n",
        "(This code is basically unchanged from the previous notebook.)"
      ],
      "metadata": {
        "id": "JvaP1DpHjI3f"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def collator(list_of_examples):\n",
        "    # Labels are simply converted into a tensor\n",
        "    batch={\n",
        "        \"labels\": torch.tensor([e[\"label\"] for e in list_of_examples])\n",
        "    }\n",
        "\n",
        "    # Examples need to be padded\n",
        "    tensors = []\n",
        "\n",
        "    # Find length of longest example\n",
        "    max_len = max(len(e[\"input_ids\"]) for e in list_of_examples)\n",
        "    max_len = max(1,max_len)\n",
        "\n",
        "    # Pad everything with zeros to length of longest example\n",
        "    for example in list_of_examples:\n",
        "        ids = torch.LongTensor(example[\"input_ids\"])\n",
        "        # pad(what,(from_left, from_right)) <- this is how we call the stock pad function\n",
        "         #pad by max - current length, pads with zero by default\n",
        "        padded = torch.nn.functional.pad(ids, (0, max_len-ids.shape[0]))\n",
        "        tensors.append(padded)\n",
        "\n",
        "    # Now that all examples are of the same length, vstack() can be used\n",
        "    # to vertically stack these into a tensor\n",
        "    batch[\"input_ids\"]=torch.vstack(tensors)\n",
        "\n",
        "    return batch"
      ],
      "metadata": {
        "id": "zXXDxNo6kwfA"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Test that out with a minimal batch of two examples, one requiring padding:"
      ],
      "metadata": {
        "id": "e7Mre52QCTtI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "batch=collator([vectorized_dataset[\"train\"][2], vectorized_dataset[\"train\"][7]])\n",
        "\n",
        "print(\"Shape of labels:\",batch[\"labels\"].shape)\n",
        "print(\"Shape of input_ids:\",batch[\"input_ids\"].shape)\n",
        "print(\"labels:\",batch[\"labels\"])\n",
        "print(\"input_ids:\",batch[\"input_ids\"])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JJnW4gwqBpF6",
        "outputId": "c462448d-6ee7-45d6-f638-a90531c3f18d"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape of labels: torch.Size([2])\n",
            "Shape of input_ids: torch.Size([2, 6])\n",
            "labels: tensor([16, 21])\n",
            "input_ids: tensor([[    1,  5427, 14216, 20003, 26070, 27863],\n",
            "        [  585,  6898, 12964, 17990,     0,     0]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# MLP model\n",
        "\n",
        "With the data now ready, we'll build the MLP model. Note that this is _identical_ to the MLP model we used for text classification: the only difference between the two applications is in the data.\n",
        "\n",
        "The model class in its simplest form has `__init__()` which instantiates the layers and `forward()` which implements the actual computation. For more information on these, please see the [PyTorch turorial](https://pytorch.org/tutorials/beginner/introyt/modelsyt_tutorial.html)."
      ],
      "metadata": {
        "id": "AOYYF5I1OWG9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# A model wants a config, I can simply inherit from the base\n",
        "# class for pretrained configs\n",
        "class MLPConfig(transformers.PretrainedConfig):\n",
        "    pass\n",
        "\n",
        "# This is the model\n",
        "class MLP(transformers.PreTrainedModel):\n",
        "\n",
        "    config_class=MLPConfig\n",
        "\n",
        "    # In the initialization method, one instantiates the layers\n",
        "    # these will be, for the most part the trained parameters of the model\n",
        "    def __init__(self, config):\n",
        "        super().__init__(config)\n",
        "\n",
        "        self.vocab_size=config.vocab_size    # embedding matrix row count\n",
        "\n",
        "        # Build and initialize embedding of vocab size +1 x hidden size\n",
        "        # (+1 because of the padding index 0!)\n",
        "        self.embedding = torch.nn.Embedding(\n",
        "            num_embeddings=self.vocab_size+1,\n",
        "            embedding_dim=config.hidden_size,\n",
        "            padding_idx=0\n",
        "        )\n",
        "\n",
        "        # Initialize the embeddings with small random values\n",
        "        torch.nn.init.uniform_(self.embedding.weight.data, -0.001, 0.001)\n",
        "        # Enforce zero values for padding\n",
        "        torch.nn.init.zeros_(self.embedding.weight.data[0,:])\n",
        "\n",
        "        # Output layer: hidden size x output size\n",
        "        self.output=torch.nn.Linear(\n",
        "            in_features=config.hidden_size,\n",
        "            out_features=config.nlabels\n",
        "        )\n",
        "\n",
        "    # The computation of the model is put into the forward() function\n",
        "    # it receives a batch of data and optionally the correct `labels`:\n",
        "    # - if given `labels`, returns (loss, output)\n",
        "    # - if not, only returns (output,)\n",
        "    def forward(self, input_ids, labels=None):\n",
        "        # 1) Look up embeddings of features, sum them up\n",
        "        embedded = self.embedding(input_ids)    # (batch, ids) -> (batch, ids, embedding_dim)\n",
        "        embedded_summed = torch.sum(embedded, dim=1)    # (batch, ids, embedding_dim) -> (batch, embedding_dim)\n",
        "\n",
        "        # NOTE: we're explicitly *not* applying a nonlinearity here to keep\n",
        "        # things linear for later analysis\n",
        "\n",
        "        # 2) Apply output layer\n",
        "        # (batch, embedding_dim) -> (batch, num_classes)\n",
        "        logits = self.output(embedded_summed)\n",
        "\n",
        "        if labels is not None:\n",
        "            # We have labels, so we ought to calculate the loss\n",
        "            loss_fn = torch.nn.CrossEntropyLoss()    # Classification loss function\n",
        "            loss = loss_fn(logits, labels)\n",
        "            return (loss, logits)\n",
        "        else:\n",
        "            # No labels, so just return the logits\n",
        "            return (logits,)"
      ],
      "metadata": {
        "id": "vP4FtrMwCpGi"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Configure the model"
      ],
      "metadata": {
        "id": "Ix6-Z0XqFGfl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "num_labels = len(POS2ID)\n",
        "\n",
        "mlp_config = MLPConfig(\n",
        "    vocab_size=len(vectorizer.vocabulary_),\n",
        "    hidden_size=20,\n",
        "    nlabels=num_labels\n",
        ")"
      ],
      "metadata": {
        "id": "I2Lwe8eoFFtw"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Train the model\n",
        "\n",
        "We will use the Hugging Face [Trainer](https://huggingface.co/docs/transformers/main_classes/trainer) class for training\n",
        "\n",
        "* Loads of arguments that control the training\n",
        "* Configurable metrics to evaluate performance\n",
        "* Data collator builds the batches\n",
        "* Early stopping callback stops when eval loss no longer improves\n",
        "* Model load/save\n",
        "* Good foundation for later deep learning course\n",
        "  "
      ],
      "metadata": {
        "id": "tdlcMObzQGGP"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "First, let's create a [`TrainingArguments`](https://huggingface.co/docs/transformers/v4.17.0/en/main_classes/trainer#transformers.TrainingArguments) object to specify hyperparameters and various other settings for training.\n",
        "\n",
        "Printing this simple dataclass object will show not only the values we set, but also the defaults for all other arguments. Don't worry if you don't understand what all of these do! Many are not relevant to us here, and you can find the details in [`Trainer` documentation](https://huggingface.co/docs/transformers/main_classes/trainer) if you are interested."
      ],
      "metadata": {
        "id": "aZgcNi4B76SX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "trainer_args = transformers.TrainingArguments(\n",
        "    \"mlp_checkpoints\", #save checkpoints here\n",
        "    evaluation_strategy=\"steps\",\n",
        "    logging_strategy=\"steps\",\n",
        "    eval_steps=500,\n",
        "    logging_steps=500,\n",
        "    learning_rate=1e-4, #learning rate of the gradient descent\n",
        "    max_steps=20000,\n",
        "    load_best_model_at_end=True,\n",
        "    per_device_train_batch_size=128\n",
        ")\n",
        "\n",
        "pprint(trainer_args)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IhfdW62z8cCn",
        "outputId": "fef21b2f-795d-44e1-925d-6919045b7f86"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "TrainingArguments(\n",
            "_n_gpu=1,\n",
            "accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True},\n",
            "adafactor=False,\n",
            "adam_beta1=0.9,\n",
            "adam_beta2=0.999,\n",
            "adam_epsilon=1e-08,\n",
            "auto_find_batch_size=False,\n",
            "bf16=False,\n",
            "bf16_full_eval=False,\n",
            "data_seed=None,\n",
            "dataloader_drop_last=False,\n",
            "dataloader_num_workers=0,\n",
            "dataloader_persistent_workers=False,\n",
            "dataloader_pin_memory=True,\n",
            "dataloader_prefetch_factor=None,\n",
            "ddp_backend=None,\n",
            "ddp_broadcast_buffers=None,\n",
            "ddp_bucket_cap_mb=None,\n",
            "ddp_find_unused_parameters=None,\n",
            "ddp_timeout=1800,\n",
            "debug=[],\n",
            "deepspeed=None,\n",
            "disable_tqdm=False,\n",
            "dispatch_batches=None,\n",
            "do_eval=True,\n",
            "do_predict=False,\n",
            "do_train=False,\n",
            "eval_accumulation_steps=None,\n",
            "eval_delay=0,\n",
            "eval_steps=500,\n",
            "evaluation_strategy=steps,\n",
            "fp16=False,\n",
            "fp16_backend=auto,\n",
            "fp16_full_eval=False,\n",
            "fp16_opt_level=O1,\n",
            "fsdp=[],\n",
            "fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},\n",
            "fsdp_min_num_params=0,\n",
            "fsdp_transformer_layer_cls_to_wrap=None,\n",
            "full_determinism=False,\n",
            "gradient_accumulation_steps=1,\n",
            "gradient_checkpointing=False,\n",
            "gradient_checkpointing_kwargs=None,\n",
            "greater_is_better=False,\n",
            "group_by_length=False,\n",
            "half_precision_backend=auto,\n",
            "hub_always_push=False,\n",
            "hub_model_id=None,\n",
            "hub_private_repo=False,\n",
            "hub_strategy=every_save,\n",
            "hub_token=<HUB_TOKEN>,\n",
            "ignore_data_skip=False,\n",
            "include_inputs_for_metrics=False,\n",
            "include_num_input_tokens_seen=False,\n",
            "include_tokens_per_second=False,\n",
            "jit_mode_eval=False,\n",
            "label_names=None,\n",
            "label_smoothing_factor=0.0,\n",
            "learning_rate=0.0001,\n",
            "length_column_name=length,\n",
            "load_best_model_at_end=True,\n",
            "local_rank=0,\n",
            "log_level=passive,\n",
            "log_level_replica=warning,\n",
            "log_on_each_node=True,\n",
            "logging_dir=mlp_checkpoints/runs/Apr02_09-00-01_72e590fe85b2,\n",
            "logging_first_step=False,\n",
            "logging_nan_inf_filter=True,\n",
            "logging_steps=500,\n",
            "logging_strategy=steps,\n",
            "lr_scheduler_kwargs={},\n",
            "lr_scheduler_type=linear,\n",
            "max_grad_norm=1.0,\n",
            "max_steps=20000,\n",
            "metric_for_best_model=loss,\n",
            "mp_parameters=,\n",
            "neftune_noise_alpha=None,\n",
            "no_cuda=False,\n",
            "num_train_epochs=3.0,\n",
            "optim=adamw_torch,\n",
            "optim_args=None,\n",
            "output_dir=mlp_checkpoints,\n",
            "overwrite_output_dir=False,\n",
            "past_index=-1,\n",
            "per_device_eval_batch_size=8,\n",
            "per_device_train_batch_size=128,\n",
            "prediction_loss_only=False,\n",
            "push_to_hub=False,\n",
            "push_to_hub_model_id=None,\n",
            "push_to_hub_organization=None,\n",
            "push_to_hub_token=<PUSH_TO_HUB_TOKEN>,\n",
            "ray_scope=last,\n",
            "remove_unused_columns=True,\n",
            "report_to=['tensorboard'],\n",
            "resume_from_checkpoint=None,\n",
            "run_name=mlp_checkpoints,\n",
            "save_on_each_node=False,\n",
            "save_only_model=False,\n",
            "save_safetensors=True,\n",
            "save_steps=500,\n",
            "save_strategy=steps,\n",
            "save_total_limit=None,\n",
            "seed=42,\n",
            "skip_memory_metrics=True,\n",
            "split_batches=None,\n",
            "tf32=None,\n",
            "torch_compile=False,\n",
            "torch_compile_backend=None,\n",
            "torch_compile_mode=None,\n",
            "torchdynamo=None,\n",
            "tpu_metrics_debug=False,\n",
            "tpu_num_cores=None,\n",
            "use_cpu=False,\n",
            "use_ipex=False,\n",
            "use_legacy_prediction_loop=False,\n",
            "use_mps_device=False,\n",
            "warmup_ratio=0.0,\n",
            "warmup_steps=0,\n",
            "weight_decay=0.0,\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Next, let's create a metric for evaluating performance during and after training. We can use the convenience function [`load_metric`](https://huggingface.co/docs/datasets/about_metrics) to load one of many pre-made metrics and wrap this for use by the trainer.\n",
        "\n",
        "We can use the basic `accuracy` metric, defined as the proportion of correctly predicted labels out of all labels. This time, though, the data is not evenly split."
      ],
      "metadata": {
        "id": "4sJwNXPU-dQX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import evaluate\n",
        "\n",
        "accuracy = evaluate.load(\"accuracy\")\n",
        "\n",
        "def compute_accuracy(outputs_and_labels):\n",
        "    outputs, labels = outputs_and_labels\n",
        "    predictions = np.argmax(outputs, axis=-1) #pick the index of the \"winning\" label\n",
        "    return accuracy.compute(predictions=predictions, references=labels)"
      ],
      "metadata": {
        "id": "u3jxIItb0BL9",
        "outputId": "b0170fa2-4949-4d00-9ffa-1831ab42efb0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "c573453f74eb408999aed626e6a34e7e",
            "ae9c1133006546579a92210597eb8a60",
            "dbccc8a6cef746e08bde588a4ca26ad9",
            "970ee40e037a4611a8415c09ba143379",
            "277b1871c444496d8f98c595e70bd439",
            "0f33be08f6084dc7bc2fa3fcdde5e0cb",
            "59b65bf171054beebfea59c63587a005",
            "8104fefb54a44c6b8f43a3ad07eab6b1",
            "ae4f5b7dba2e448fab44fb5ddd5462dc",
            "d049c7a037d549fe8fbedccd46873129",
            "96bbece15f4b4b64b5ce2b7fdba8ac50"
          ]
        }
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/4.20k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c573453f74eb408999aed626e6a34e7e"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can then create the `Trainer` and train the model by invoking the [`Trainer.train`](https://huggingface.co/docs/transformers/main_classes/trainer#transformers.Trainer.train) function.\n",
        "\n",
        "In addition to the model, the settings passed in through the `TrainingArguments` object created above (`trainer_args`), the data, and the metric defined above, we create and pass the following to the `Trainer`:\n",
        "\n",
        "* [data collator](https://huggingface.co/docs/transformers/main_classes/data_collator): groups input into batches\n",
        "* [`EarlyStoppingCallback`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.EarlyStoppingCallback): stops training when performance stops improving"
      ],
      "metadata": {
        "id": "S7kbz8uU-zpQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Make a new model\n",
        "mlp = MLP(mlp_config)\n",
        "\n",
        "\n",
        "# Argument gives the number of steps of patience before early stopping\n",
        "# i.e. training is stopped when the evaluation loss fails to improve\n",
        "# certain number of times\n",
        "early_stopping = transformers.EarlyStoppingCallback(5)\n",
        "\n",
        "trainer = transformers.Trainer(\n",
        "    model=mlp,\n",
        "    args=trainer_args,\n",
        "    train_dataset=vectorized_dataset[\"train\"],\n",
        "    eval_dataset=vectorized_dataset[\"validation\"],\n",
        "    compute_metrics=compute_accuracy,\n",
        "    data_collator=collator,\n",
        "    callbacks=[early_stopping]\n",
        ")\n",
        "\n",
        "trainer.train()"
      ],
      "metadata": {
        "id": "AoEoWsj4P_zN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "02ed0bf3-742e-4a6d-cdbd-316205ecc54a"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:432: FutureWarning: Passing the following arguments to `Accelerator` is deprecated and will be removed in version 1.0 of Accelerate: dict_keys(['dispatch_batches', 'split_batches', 'even_batches', 'use_seedable_sampler']). Please pass an `accelerate.DataLoaderConfiguration` instead: \n",
            "dataloader_config = DataLoaderConfiguration(dispatch_batches=None, split_batches=False, even_batches=True, use_seedable_sampler=True)\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='20000' max='20000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [20000/20000 08:55, Epoch 12/13]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>3.748200</td>\n",
              "      <td>3.609401</td>\n",
              "      <td>0.418987</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>3.392700</td>\n",
              "      <td>3.161311</td>\n",
              "      <td>0.526362</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>2.907600</td>\n",
              "      <td>2.676499</td>\n",
              "      <td>0.569662</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>2.449400</td>\n",
              "      <td>2.287558</td>\n",
              "      <td>0.586095</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>2.112600</td>\n",
              "      <td>2.002587</td>\n",
              "      <td>0.606772</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>1.854400</td>\n",
              "      <td>1.787015</td>\n",
              "      <td>0.629901</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3500</td>\n",
              "      <td>1.663300</td>\n",
              "      <td>1.619324</td>\n",
              "      <td>0.654277</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4000</td>\n",
              "      <td>1.497800</td>\n",
              "      <td>1.481121</td>\n",
              "      <td>0.680328</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4500</td>\n",
              "      <td>1.383800</td>\n",
              "      <td>1.366766</td>\n",
              "      <td>0.701881</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5000</td>\n",
              "      <td>1.267100</td>\n",
              "      <td>1.270650</td>\n",
              "      <td>0.720143</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5500</td>\n",
              "      <td>1.179000</td>\n",
              "      <td>1.189274</td>\n",
              "      <td>0.735349</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6000</td>\n",
              "      <td>1.102800</td>\n",
              "      <td>1.119746</td>\n",
              "      <td>0.749075</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6500</td>\n",
              "      <td>1.027300</td>\n",
              "      <td>1.060060</td>\n",
              "      <td>0.760251</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7000</td>\n",
              "      <td>0.973400</td>\n",
              "      <td>1.008298</td>\n",
              "      <td>0.771641</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7500</td>\n",
              "      <td>0.925900</td>\n",
              "      <td>0.963439</td>\n",
              "      <td>0.781959</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8000</td>\n",
              "      <td>0.887700</td>\n",
              "      <td>0.923759</td>\n",
              "      <td>0.789630</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8500</td>\n",
              "      <td>0.834700</td>\n",
              "      <td>0.889231</td>\n",
              "      <td>0.795958</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9000</td>\n",
              "      <td>0.814500</td>\n",
              "      <td>0.858592</td>\n",
              "      <td>0.802850</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9500</td>\n",
              "      <td>0.780700</td>\n",
              "      <td>0.831285</td>\n",
              "      <td>0.808866</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10000</td>\n",
              "      <td>0.748300</td>\n",
              "      <td>0.807194</td>\n",
              "      <td>0.813286</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10500</td>\n",
              "      <td>0.726000</td>\n",
              "      <td>0.785628</td>\n",
              "      <td>0.818348</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11000</td>\n",
              "      <td>0.709500</td>\n",
              "      <td>0.766311</td>\n",
              "      <td>0.822223</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11500</td>\n",
              "      <td>0.688300</td>\n",
              "      <td>0.749063</td>\n",
              "      <td>0.825766</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12000</td>\n",
              "      <td>0.676600</td>\n",
              "      <td>0.733447</td>\n",
              "      <td>0.828979</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12500</td>\n",
              "      <td>0.644300</td>\n",
              "      <td>0.719618</td>\n",
              "      <td>0.831451</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13000</td>\n",
              "      <td>0.647400</td>\n",
              "      <td>0.707103</td>\n",
              "      <td>0.834333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13500</td>\n",
              "      <td>0.620900</td>\n",
              "      <td>0.695915</td>\n",
              "      <td>0.836611</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14000</td>\n",
              "      <td>0.614400</td>\n",
              "      <td>0.685981</td>\n",
              "      <td>0.838636</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14500</td>\n",
              "      <td>0.606600</td>\n",
              "      <td>0.677056</td>\n",
              "      <td>0.840680</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15000</td>\n",
              "      <td>0.597300</td>\n",
              "      <td>0.669202</td>\n",
              "      <td>0.842315</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15500</td>\n",
              "      <td>0.593300</td>\n",
              "      <td>0.662232</td>\n",
              "      <td>0.843717</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16000</td>\n",
              "      <td>0.574200</td>\n",
              "      <td>0.656109</td>\n",
              "      <td>0.844866</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16500</td>\n",
              "      <td>0.570200</td>\n",
              "      <td>0.650808</td>\n",
              "      <td>0.846092</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17000</td>\n",
              "      <td>0.570200</td>\n",
              "      <td>0.646321</td>\n",
              "      <td>0.846988</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17500</td>\n",
              "      <td>0.567600</td>\n",
              "      <td>0.642583</td>\n",
              "      <td>0.847475</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18000</td>\n",
              "      <td>0.559400</td>\n",
              "      <td>0.639558</td>\n",
              "      <td>0.848195</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18500</td>\n",
              "      <td>0.561700</td>\n",
              "      <td>0.637198</td>\n",
              "      <td>0.848623</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19000</td>\n",
              "      <td>0.551300</td>\n",
              "      <td>0.635538</td>\n",
              "      <td>0.849130</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19500</td>\n",
              "      <td>0.551500</td>\n",
              "      <td>0.634555</td>\n",
              "      <td>0.849324</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20000</td>\n",
              "      <td>0.552700</td>\n",
              "      <td>0.634221</td>\n",
              "      <td>0.849363</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=20000, training_loss=1.0933606048583984, metrics={'train_runtime': 537.2549, 'train_samples_per_second': 4764.963, 'train_steps_per_second': 37.226, 'total_flos': 120764047446.0, 'train_loss': 1.0933606048583984, 'epoch': 12.57})"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can then evaluate the trained model on a given dataset (here our test subset) by calling [`Trainer.evaluate`](https://huggingface.co/docs/transformers/main_classes/trainer#transformers.Trainer.evaluate):"
      ],
      "metadata": {
        "id": "Td03fcIa-6Hj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "eval_results = trainer.evaluate(vectorized_dataset[\"test\"])\n",
        "\n",
        "print(\"Accuracy:\", eval_results[\"eval_accuracy\"])"
      ],
      "metadata": {
        "id": "9nlEwpnF2Vow",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "934b273c-ca51-4589-b407-b5fdfc29f237"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='5805' max='5805' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [5805/5805 00:08]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy: 0.855540002153548\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "That's pretty poor performance for a task as simple as POS tagging where state-of-the-art accuracies are generally 97-99%. (The approach demonstrated in this notebook should be considered more of a teaching tool than a serious tagger implementation.)\n",
        "\n",
        "However, the result is certainly much better than random, so we can conclude that the model is learning something about the task."
      ],
      "metadata": {
        "id": "pEbNfGDhfdXK"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Save model for later use\n",
        "\n",
        "* You can save it with `trainer.save_model()`\n",
        "* You can load it with `MLP.from_pretrained()`\n"
      ],
      "metadata": {
        "id": "P0OYB3TRp-IH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.save_model(\"mlp-postagger\")"
      ],
      "metadata": {
        "id": "FosHTDw3p9dd"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# What has the model learned?\n",
        "\n",
        "* The embeddings should have some meaning to them\n",
        "* Similar features should have similar embeddings"
      ],
      "metadata": {
        "id": "13aB7DuqzeFi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Grab the embedding matrix out of the trained model\n",
        "# and drop the first row (padding 0)\n",
        "# then we can treat the embeddings as vectors\n",
        "\n",
        "weights=mlp.embedding.weight.detach().cpu().numpy()\n",
        "weights=weights[1:,:]"
      ],
      "metadata": {
        "id": "M6TUrVkMCmz7"
      },
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "qry_idx=vectorizer.vocabulary_[\"token[0]=in\"]\n",
        "\n",
        "#calculate the distance of the \"in\" embedding to all other embeddings\n",
        "distance_to_qry=sklearn.metrics.pairwise.euclidean_distances(weights[qry_idx:qry_idx+1,:],weights)\n",
        "nearest_neighbors=np.argsort(distance_to_qry) #indices of words nearest to \"in\"\n",
        "for nearest in nearest_neighbors[0,:20]:\n",
        "    print(idx2feat[nearest])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4uTTvRgGDU9D",
        "outputId": "84fda8df-3c44-47cb-e78f-7a95b45fd168"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "token[0]=in\n",
            "token[0]=at\n",
            "token[0]=with\n",
            "token[0]=from\n",
            "token[0]=by\n",
            "token[0]=for\n",
            "token[0]=on\n",
            "token[0]=after\n",
            "token[0]=of\n",
            "token[0]=In\n",
            "token[0]=as\n",
            "token[0]=into\n",
            "token[0]=under\n",
            "token[0]=than\n",
            "token[0]=between\n",
            "token[0]=against\n",
            "token[0]=over\n",
            "token[0]=since\n",
            "token[0]=before\n",
            "token[0]=through\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "* The embeddings indeed seem to reflect the task and capture aspects of the meaning of words relevant to the task\n",
        "* But now we have many classes, so we should take that into account too\n",
        "* We can take the dot-product of the feature embeddings with the output layer weight of the class we care about\n",
        "* When you think how the information propagates in the network, this will give us a single number reflecting each feature w.r.t. the selected label\n",
        "* Technically speaking, it is the prediction of an example which only has that one feature, with respect to that one class\n",
        "* Here is how we can implement it (here we rely on the fact that the model is linear, since we didn't include a nonlinearity earlier in the model's `forward()`"
      ],
      "metadata": {
        "id": "cQCmThdu2LDR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy\n",
        "\n",
        "embedding_weights=weights    #shape (features, embedding-dim)\n",
        "output_weights=mlp.output.weight.detach().cpu().numpy()    #shape (num-labels, embedding-dim)\n",
        "\n",
        "# We just matrix-multiply these together, since this gives us all the dot-products\n",
        "weights_by_label=numpy.matmul(embedding_weights, output_weights.T)\n",
        "weights_by_label.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T5coMII4ABPt",
        "outputId": "d5ef86e9-02d0-45e0-ab82-00582f08e8ee"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(30000, 47)"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_most_important_features_for_and_against(label):\n",
        "    label_idx = POS2ID[label]\n",
        "    feature_weights = weights_by_label[:,label_idx] #pick the column that interests us\n",
        "\n",
        "    #The shape of feature_weights is (feature_vocab_size,) i.e. it is a vector\n",
        "    features_weight_idx = numpy.argsort(-feature_weights) #sort in descending order, this will be vector of indices\n",
        "    features_for = [idx2feat[feature_idx] for feature_idx in features_weight_idx[:20]]\n",
        "    features_against = [idx2feat[feature_idx] for feature_idx in features_weight_idx[-20:][::-1]]\n",
        "    return features_for, features_against\n",
        "\n",
        "for label in (\"DT\", \"NN\", \"VB\"):\n",
        "    dt_plus,dt_minus=get_most_important_features_for_and_against(label)\n",
        "    print(f\"{label}: {POS2DESCRIPTION[label]}\")\n",
        "    print(f\"Most important features *for* label {label}:\")\n",
        "    pprint(\"   \".join(dt_plus))\n",
        "    print()\n",
        "    print(f\"Most important features *against* label {label}:\")\n",
        "    pprint(\"   \".join(dt_minus))\n",
        "    print(\"\\n------\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4vnJj7cNBagc",
        "outputId": "f49e5a94-fe89-4e0c-c8cf-be71a2274f0f"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "DT: Determiner\n",
            "Most important features *for* label DT:\n",
            "('token[0]=the   token[0]=a   token[0]=The   token[0]=an   token[0]=this   '\n",
            " 'token[0]=A   token[0]=some   token[0]=no   token[0]=any   token[0]=all   '\n",
            " 'token[0]=both   token[0]=another   token[0]=those   token[0]=This   '\n",
            " 'token[0]=and   token[0]=No   token[0]=An   token[0]=these   token[0]=to   '\n",
            " 'token[0]=each')\n",
            "\n",
            "Most important features *against* label DT:\n",
            "('token[0]=is   token[0]=has   token[0]=U.S.   token[0]=Thursday   '\n",
            " 'token[0]=Wednesday   token[0]=his   token[0]=said   token[0]=Tuesday   '\n",
            " 'token[0]=Monday   token[0]=was   token[0]=3.   token[0]=Friday   '\n",
            " 'token[0]=Sunday   token[0]=were   token[0]=their   token[0]=its   '\n",
            " 'token[0]=England   token[0]=Minister   token[0]=President   '\n",
            " 'token[0]=Saturday')\n",
            "\n",
            "------\n",
            "\n",
            "NN: Noun, singular or mass\n",
            "Most important features *for* label NN:\n",
            "('token[0]=percent   token[0]=government   token[0]=year   token[0]=SOCCER   '\n",
            " 'token[0]=police   token[0]=week   token[0]=time   token[0]=state   '\n",
            " 'token[0]=company   token[0]=group   token[0]=market   token[0]=spokesman   '\n",
            " 'token[-1]=a   token[1]=of   token[0]=home   token[0]=world   '\n",
            " 'token[0]=division   token[0]=president   token[-1]=The   token[0]=peace')\n",
            "\n",
            "Most important features *against* label NN:\n",
            "('token[0]=who   token[0]=-   token[0]=:   token[0]=$   token[0]=which   '\n",
            " 'token[0]=it   token[0]=he   token[0]=his   token[0]=be   token[0]=--   '\n",
            " 'token[0]=their   token[0]=and   token[0]=they   token[0]=her   '\n",
            " 'token[0]=its   token[-1]=will   token[0]=more   token[0]=most   token[0]=I   '\n",
            " 'token[0]=when')\n",
            "\n",
            "------\n",
            "\n",
            "VB: Verb, base form\n",
            "Most important features *for* label VB:\n",
            "('token[0]=be   token[-1]=to   token[-1]=will   token[-1]=would   '\n",
            " 'token[0]=beat   token[0]=\"   token[-1]=could   token[0]=take   '\n",
            " \"token[-2]=did   token[-1]=should   token[-1]=n't   token[-1]=not   \"\n",
            " 'token[0]=meet   token[-1]=can   token[0]=go   token[0]=make   token[0]=it   '\n",
            " 'token[0]=he   token[0]=(   token[-2]=do')\n",
            "\n",
            "Most important features *against* label VB:\n",
            "('token[0]=was   token[0]=said   token[0]=were   token[0]=had   token[0]=not   '\n",
            " 'token[0]=and   token[0]=.   token[0]=,   token[0]=SOCCER   token[0]=also   '\n",
            " 'token[0]=percent   token[0]=year   token[0]=one   token[0]=two   '\n",
            " 'token[0]=are   token[0]=0   token[0]=government   token[0]=1   '\n",
            " 'token[0]=Men   token[0]=but')\n",
            "\n",
            "------\n",
            "\n"
          ]
        }
      ]
    }
  ]
}