mlp_imdb_hf_dset_and_trainer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "eeb454382a57454893131f6521914a73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_19d1d9e90c5b4430ab42f323c4049c35",
              "IPY_MODEL_0a0b6fbdf84f4b48b9c2ef49a0250404",
              "IPY_MODEL_1a56cbd49d384c519b64714fea68a641"
            ],
            "layout": "IPY_MODEL_5de8dbc30dcc46bf91c6b06a4e13931b"
          }
        },
        "19d1d9e90c5b4430ab42f323c4049c35": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_218dc6efe29c4b6abddf064f6d5658ed",
            "placeholder": "​",
            "style": "IPY_MODEL_16b02cc48b71420a9ddad7185bb3e029",
            "value": "Downloading readme: 100%"
          }
        },
        "0a0b6fbdf84f4b48b9c2ef49a0250404": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c68fd158757d423585d58956d1a99eaf",
            "max": 7809,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_222daa34d70b40388521329453f1f8dd",
            "value": 7809
          }
        },
        "1a56cbd49d384c519b64714fea68a641": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_54733c51d16d48f098862efe3b5981a6",
            "placeholder": "​",
            "style": "IPY_MODEL_e39eabb6495a47b08f1cf12abe3be236",
            "value": " 7.81k/7.81k [00:00&lt;00:00, 160kB/s]"
          }
        },
        "5de8dbc30dcc46bf91c6b06a4e13931b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "218dc6efe29c4b6abddf064f6d5658ed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "16b02cc48b71420a9ddad7185bb3e029": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c68fd158757d423585d58956d1a99eaf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "222daa34d70b40388521329453f1f8dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "54733c51d16d48f098862efe3b5981a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e39eabb6495a47b08f1cf12abe3be236": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d1a81417dae24a53a8b53c47e6b4381c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_36f0168cb84a4269bbcb22c73ab44d54",
              "IPY_MODEL_4e60b02fa07540cc81b3d8fc5101bc52",
              "IPY_MODEL_63ada6281f1d448b866e6068020f40ac"
            ],
            "layout": "IPY_MODEL_2d45cf6772d149bfafbd014365f8f191"
          }
        },
        "36f0168cb84a4269bbcb22c73ab44d54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7f80aad5a1bc443ea5c0c1b5ae871d87",
            "placeholder": "​",
            "style": "IPY_MODEL_5cd8a219b69e46aa9913ecf9ac16d8b5",
            "value": "Downloading data: 100%"
          }
        },
        "4e60b02fa07540cc81b3d8fc5101bc52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ffcb6442757c4d8fa0b64d6813c261f2",
            "max": 20979968,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0ea1ef2cd8574f3385da830711032285",
            "value": 20979968
          }
        },
        "63ada6281f1d448b866e6068020f40ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bfd298f36bea418b8e80000245c25ed1",
            "placeholder": "​",
            "style": "IPY_MODEL_2a0634254c42464e960eca2fd4f185a3",
            "value": " 21.0M/21.0M [00:00&lt;00:00, 39.3MB/s]"
          }
        },
        "2d45cf6772d149bfafbd014365f8f191": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7f80aad5a1bc443ea5c0c1b5ae871d87": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5cd8a219b69e46aa9913ecf9ac16d8b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ffcb6442757c4d8fa0b64d6813c261f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0ea1ef2cd8574f3385da830711032285": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bfd298f36bea418b8e80000245c25ed1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2a0634254c42464e960eca2fd4f185a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9c98cd7ac7e340a2830fe677bf5f60cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_28ea4db6f6d44017b326ce9da7830ca4",
              "IPY_MODEL_288ac80ae11d421bac87e2438e62c2b5",
              "IPY_MODEL_d686c7ce5f32456bb071c49d772678d0"
            ],
            "layout": "IPY_MODEL_9a8e21f649e440cda780ffca8d67805c"
          }
        },
        "28ea4db6f6d44017b326ce9da7830ca4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b5eebda55aad4a48889c8e7cf3ab4975",
            "placeholder": "​",
            "style": "IPY_MODEL_f32cbba9a7b6489a831f9a6bea21938c",
            "value": "Downloading data: 100%"
          }
        },
        "288ac80ae11d421bac87e2438e62c2b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d35803ce3faf4011ade3032954d3fc49",
            "max": 20470363,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f81a4981e95941c3a915b56f9e2f1ea0",
            "value": 20470363
          }
        },
        "d686c7ce5f32456bb071c49d772678d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2c84144d7ea8458fafb216b32d1d6d3e",
            "placeholder": "​",
            "style": "IPY_MODEL_13e4f6d9a4be47209385c67158a45bb3",
            "value": " 20.5M/20.5M [00:00&lt;00:00, 36.8MB/s]"
          }
        },
        "9a8e21f649e440cda780ffca8d67805c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b5eebda55aad4a48889c8e7cf3ab4975": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f32cbba9a7b6489a831f9a6bea21938c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d35803ce3faf4011ade3032954d3fc49": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f81a4981e95941c3a915b56f9e2f1ea0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2c84144d7ea8458fafb216b32d1d6d3e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "13e4f6d9a4be47209385c67158a45bb3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6a6bd6514aa8402e978923dde155551e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a380327b8df24a01b85511c64523ca01",
              "IPY_MODEL_bcc23dfef01e42fea8bb94b0c5e6f3fc",
              "IPY_MODEL_aacad03af7d34da481016a5a2c801bc7"
            ],
            "layout": "IPY_MODEL_9dd07ae5bb424ebdadf42d9e10196f8f"
          }
        },
        "a380327b8df24a01b85511c64523ca01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cf4e654be55140b1ad6d9f5439081ee9",
            "placeholder": "​",
            "style": "IPY_MODEL_a0ec090f1b154f328f4aa216370ebdce",
            "value": "Downloading data: 100%"
          }
        },
        "bcc23dfef01e42fea8bb94b0c5e6f3fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_959de368cabf4d51b46825f895b88b97",
            "max": 41996509,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_45eb295c8b9b413fae926166cd36836c",
            "value": 41996509
          }
        },
        "aacad03af7d34da481016a5a2c801bc7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_466f13d171a64be3a5055424d1d71208",
            "placeholder": "​",
            "style": "IPY_MODEL_a4206c82275847ef8c464265c9eaea08",
            "value": " 42.0M/42.0M [00:00&lt;00:00, 46.6MB/s]"
          }
        },
        "9dd07ae5bb424ebdadf42d9e10196f8f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cf4e654be55140b1ad6d9f5439081ee9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a0ec090f1b154f328f4aa216370ebdce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "959de368cabf4d51b46825f895b88b97": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "45eb295c8b9b413fae926166cd36836c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "466f13d171a64be3a5055424d1d71208": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a4206c82275847ef8c464265c9eaea08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "90eb5c7a2d8f4627bcaae84cfc8f4766": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c1521782dbe74f9bac8cda73d2c4f3fb",
              "IPY_MODEL_fe4208bf6680465697a87889ec3f120c",
              "IPY_MODEL_5cfdb87a29b34693b2b65fb915bd4dd6"
            ],
            "layout": "IPY_MODEL_128df31502d445159c9b82651e01014f"
          }
        },
        "c1521782dbe74f9bac8cda73d2c4f3fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_832baac0e0af4bccbc5af062076bcf96",
            "placeholder": "​",
            "style": "IPY_MODEL_54748003511e4ff49a91c73d3f3df2f6",
            "value": "Generating train split: 100%"
          }
        },
        "fe4208bf6680465697a87889ec3f120c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8bc478ee4f37434a9e92e28a90c659ea",
            "max": 25000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cffa2115c14f40c5be8c0cee67193b49",
            "value": 25000
          }
        },
        "5cfdb87a29b34693b2b65fb915bd4dd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cb314a81ab75474fb481c52f4a1ed842",
            "placeholder": "​",
            "style": "IPY_MODEL_e36d5e5b22e0407b810231a97acbf73e",
            "value": " 25000/25000 [00:00&lt;00:00, 66711.86 examples/s]"
          }
        },
        "128df31502d445159c9b82651e01014f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "832baac0e0af4bccbc5af062076bcf96": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "54748003511e4ff49a91c73d3f3df2f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8bc478ee4f37434a9e92e28a90c659ea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cffa2115c14f40c5be8c0cee67193b49": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cb314a81ab75474fb481c52f4a1ed842": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e36d5e5b22e0407b810231a97acbf73e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "61bd816155a946c9b07a864c22231f85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_137927bd797149efb17707da77674f78",
              "IPY_MODEL_41d45ef201984254877e67128611b00f",
              "IPY_MODEL_9652d4067a9344edb35777d73a140e54"
            ],
            "layout": "IPY_MODEL_44b7a2315a124ed6ae7d01eb9d41e87d"
          }
        },
        "137927bd797149efb17707da77674f78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_79a8c323bee44560bd087a1f052cb1af",
            "placeholder": "​",
            "style": "IPY_MODEL_593301961d294c0aa115fe23699fd8e7",
            "value": "Generating test split: 100%"
          }
        },
        "41d45ef201984254877e67128611b00f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2bfb7e0fb6b744b1a1ab281a807e3511",
            "max": 25000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b12b0d90ed1840cda3aa64d36f2f3d7c",
            "value": 25000
          }
        },
        "9652d4067a9344edb35777d73a140e54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_599c8f9597aa4d7cbf08fe8740a9841b",
            "placeholder": "​",
            "style": "IPY_MODEL_acd06f252bca41509d20ca29dd2b8b86",
            "value": " 25000/25000 [00:00&lt;00:00, 79754.75 examples/s]"
          }
        },
        "44b7a2315a124ed6ae7d01eb9d41e87d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "79a8c323bee44560bd087a1f052cb1af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "593301961d294c0aa115fe23699fd8e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2bfb7e0fb6b744b1a1ab281a807e3511": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b12b0d90ed1840cda3aa64d36f2f3d7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "599c8f9597aa4d7cbf08fe8740a9841b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "acd06f252bca41509d20ca29dd2b8b86": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bf459c020258434dbc54abd2761c95fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ff7a54f4cdcf4c8f9d82df85217c8309",
              "IPY_MODEL_bc9291dba526497fb31cf354685ec8af",
              "IPY_MODEL_8f3ece1cf54b4e7f83c5413bf359e9e3"
            ],
            "layout": "IPY_MODEL_cc1efb5daac348ba87eb2e52e5c4a776"
          }
        },
        "ff7a54f4cdcf4c8f9d82df85217c8309": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b364405e003e49afb820eac3827d422d",
            "placeholder": "​",
            "style": "IPY_MODEL_3d87aaa4f44148839e84d32f7292422e",
            "value": "Generating unsupervised split: 100%"
          }
        },
        "bc9291dba526497fb31cf354685ec8af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2e40d6d0905341f295f6e2563741783a",
            "max": 50000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9ac126c24753423698adc9d6c5b48087",
            "value": 50000
          }
        },
        "8f3ece1cf54b4e7f83c5413bf359e9e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3c6ec360930e4a57931cca59cceb136e",
            "placeholder": "​",
            "style": "IPY_MODEL_ca2557e24b204e9abc13522c5c192e91",
            "value": " 50000/50000 [00:00&lt;00:00, 78904.33 examples/s]"
          }
        },
        "cc1efb5daac348ba87eb2e52e5c4a776": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b364405e003e49afb820eac3827d422d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3d87aaa4f44148839e84d32f7292422e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2e40d6d0905341f295f6e2563741783a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9ac126c24753423698adc9d6c5b48087": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3c6ec360930e4a57931cca59cceb136e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ca2557e24b204e9abc13522c5c192e91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0e858b1ccaef4fd18be0b4079e4c8646": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1094731701564aee869d72736d8f9a88",
              "IPY_MODEL_cdc28da3728a49f8915dabf766dc26a8",
              "IPY_MODEL_ccf998160452482fa9d67af801e9d169"
            ],
            "layout": "IPY_MODEL_c75541bed1794a6bb05890ca783e0bb1"
          }
        },
        "1094731701564aee869d72736d8f9a88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_124e307a758c406684bfc6da8997a7d3",
            "placeholder": "​",
            "style": "IPY_MODEL_2fb1b9bb43d84dd3aacd4589fc2aada4",
            "value": "Map (num_proc=4): 100%"
          }
        },
        "cdc28da3728a49f8915dabf766dc26a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bb1b8547de064474a0c18bf66cce7889",
            "max": 25000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c3a1eebd59d44f9db9e351eeabc79a9a",
            "value": 25000
          }
        },
        "ccf998160452482fa9d67af801e9d169": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d6b3ac69a446496eafc88a80347a5393",
            "placeholder": "​",
            "style": "IPY_MODEL_9a4b67c0fb774ac7b147098e1efd7260",
            "value": " 25000/25000 [00:13&lt;00:00, 1836.14 examples/s]"
          }
        },
        "c75541bed1794a6bb05890ca783e0bb1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "124e307a758c406684bfc6da8997a7d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2fb1b9bb43d84dd3aacd4589fc2aada4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bb1b8547de064474a0c18bf66cce7889": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c3a1eebd59d44f9db9e351eeabc79a9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d6b3ac69a446496eafc88a80347a5393": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9a4b67c0fb774ac7b147098e1efd7260": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "22941c5f34504afd834c27bc7b14467d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cb72facec89145b5bc20444949ac3200",
              "IPY_MODEL_957fa9d3dc3344faa6b28534b3051e19",
              "IPY_MODEL_e758295d086a495e99fca994d7509ad3"
            ],
            "layout": "IPY_MODEL_8b1014a383764fe6b5e5c0c341b5e8f5"
          }
        },
        "cb72facec89145b5bc20444949ac3200": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f622826c23b647c78671cf589f8b389e",
            "placeholder": "​",
            "style": "IPY_MODEL_b01b1e69045c47f5a743821f14d9d970",
            "value": "Map (num_proc=4): 100%"
          }
        },
        "957fa9d3dc3344faa6b28534b3051e19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_15dd150da71c4818ade5f27be603a147",
            "max": 25000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b9b95ed802084d41b4975bf5169ea9ae",
            "value": 25000
          }
        },
        "e758295d086a495e99fca994d7509ad3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7129ce8bc63745ad9605fc60bf086c3a",
            "placeholder": "​",
            "style": "IPY_MODEL_c2d63cd124644f59a7dabfc69f790948",
            "value": " 25000/25000 [00:12&lt;00:00, 1108.05 examples/s]"
          }
        },
        "8b1014a383764fe6b5e5c0c341b5e8f5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f622826c23b647c78671cf589f8b389e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b01b1e69045c47f5a743821f14d9d970": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "15dd150da71c4818ade5f27be603a147": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b9b95ed802084d41b4975bf5169ea9ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7129ce8bc63745ad9605fc60bf086c3a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c2d63cd124644f59a7dabfc69f790948": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c29e659569f141229de8e4a25ccc00f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_01d2e73d04914715b58c7b2227e5defe",
              "IPY_MODEL_6b8ec5013a5b4fbabfe2a9a63777a3e5",
              "IPY_MODEL_1445fa6563764220a29cbd7d30045d2c"
            ],
            "layout": "IPY_MODEL_bcf5048367eb46bd97f800049a0a164c"
          }
        },
        "01d2e73d04914715b58c7b2227e5defe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3bb2265a872a40cb9704dee3cf5e8858",
            "placeholder": "​",
            "style": "IPY_MODEL_189c4e0f87a64e81a62f92ef431d133b",
            "value": "Downloading builder script: 100%"
          }
        },
        "6b8ec5013a5b4fbabfe2a9a63777a3e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9b8c5af3524146e2ad367d7b2ba65486",
            "max": 4203,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ba26f013e85341eca873279c5dafbdf3",
            "value": 4203
          }
        },
        "1445fa6563764220a29cbd7d30045d2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2acbffae3c3f44428c8cdc494004cd5e",
            "placeholder": "​",
            "style": "IPY_MODEL_ccf5918a328f44da9a5d061b46cb89ef",
            "value": " 4.20k/4.20k [00:00&lt;00:00, 177kB/s]"
          }
        },
        "bcf5048367eb46bd97f800049a0a164c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3bb2265a872a40cb9704dee3cf5e8858": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "189c4e0f87a64e81a62f92ef431d133b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9b8c5af3524146e2ad367d7b2ba65486": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ba26f013e85341eca873279c5dafbdf3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2acbffae3c3f44428c8cdc494004cd5e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ccf5918a328f44da9a5d061b46cb89ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TurkuNLP/intro-to-nlp/blob/master/mlp_imdb_hf_dset_and_trainer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Setup\n",
        "\n",
        "Before we start running our own Python code, install the required Python packages using [pip](https://en.wikipedia.org/wiki/Pip):\n",
        "\n",
        "* [`transformers`](https://huggingface.co/docs/transformers/index) is a popular deep learning package primarily on top of torch, we need to reinstall it with the [torch] configuration (might take a substantial amount of time)\n",
        "* [`datasets`](https://huggingface.co/docs/datasets/) provides support for loading, creating, and manipulating datasets\n",
        "* evaluate is a library of performance metrics (like accuracy etc)\n",
        "\n",
        "**You will likely need to do a Runtime/Restart session for everything to work after the installation.**"
      ],
      "metadata": {
        "id": "LDZZUKzfPRWT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip3 install -q datasets evaluate\n",
        "!pip install transformers[torch]"
      ],
      "metadata": {
        "id": "fKPHBYptQDsK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c73a989e-ed35-475a-90a8-6aeac927fc65"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m510.5/510.5 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m13.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m10.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: transformers[torch] in /usr/local/lib/python3.10/dist-packages (4.38.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.20.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (1.25.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (24.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2023.12.25)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.15.2)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.4.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (4.66.2)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2.2.1+cu121)\n",
            "Collecting accelerate>=0.21.0 (from transformers[torch])\n",
            "  Downloading accelerate-0.28.0-py3-none-any.whl (290 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m290.1/290.1 kB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->transformers[torch]) (5.9.5)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers[torch]) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers[torch]) (4.10.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (3.1.3)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch->transformers[torch])\n",
            "  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m37.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch->transformers[torch])\n",
            "  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m35.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch->transformers[torch])\n",
            "  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m15.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch->transformers[torch])\n",
            "  Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cublas-cu12==12.1.3.1 (from torch->transformers[torch])\n",
            "  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch->transformers[torch])\n",
            "  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch->transformers[torch])\n",
            "  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch->transformers[torch])\n",
            "  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch->transformers[torch])\n",
            "  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nccl-cu12==2.19.3 (from torch->transformers[torch])\n",
            "  Downloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h\u001b[33mWARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProtocolError('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))': /simple/nvidia-nvtx-cu12/\u001b[0m\u001b[33m\n",
            "\u001b[0mCollecting nvidia-nvtx-cu12==12.1.105 (from torch->transformers[torch])\n",
            "  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (2.2.0)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch->transformers[torch])\n",
            "  Downloading nvidia_nvjitlink_cu12-12.4.99-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.1/21.1 MB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (2024.2.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->transformers[torch]) (2.1.5)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->transformers[torch]) (1.3.0)\n",
            "Installing collected packages: nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, accelerate\n",
            "Successfully installed accelerate-0.28.0 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.4.99 nvidia-nvtx-cu12-12.1.105\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "(Above, the `!` at the start of the line tells the notebook to run the line as an operating system command rather than Python code, and the `-q` argument to `pip` runs the command in \"quiet\" mode, with less output.)"
      ],
      "metadata": {
        "id": "xOeI-LA9RcYL"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Get and prepare data\n",
        "\n",
        "*   Let us work with the IMDB dataset of movie review sentiment\n",
        "*   25,000 positive reviews\n",
        "*   25,000 negative reviews\n",
        "*   50,000 unlabeled reviews (which we discard for the time being)\n"
      ],
      "metadata": {
        "id": "_fCdfQfNNzwt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pprint import pprint #pprint => pretty-print, I use it occassionally throughout the notebook\n",
        "import datasets\n",
        "import torch\n",
        "dset=datasets.load_dataset(\"imdb\")\n",
        "pprint(dset)"
      ],
      "metadata": {
        "id": "QxmgHoKDTN2_",
        "outputId": "b94c7f5d-9008-4d92-a288-5ef1c1906c4b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 608,
          "referenced_widgets": [
            "eeb454382a57454893131f6521914a73",
            "19d1d9e90c5b4430ab42f323c4049c35",
            "0a0b6fbdf84f4b48b9c2ef49a0250404",
            "1a56cbd49d384c519b64714fea68a641",
            "5de8dbc30dcc46bf91c6b06a4e13931b",
            "218dc6efe29c4b6abddf064f6d5658ed",
            "16b02cc48b71420a9ddad7185bb3e029",
            "c68fd158757d423585d58956d1a99eaf",
            "222daa34d70b40388521329453f1f8dd",
            "54733c51d16d48f098862efe3b5981a6",
            "e39eabb6495a47b08f1cf12abe3be236",
            "d1a81417dae24a53a8b53c47e6b4381c",
            "36f0168cb84a4269bbcb22c73ab44d54",
            "4e60b02fa07540cc81b3d8fc5101bc52",
            "63ada6281f1d448b866e6068020f40ac",
            "2d45cf6772d149bfafbd014365f8f191",
            "7f80aad5a1bc443ea5c0c1b5ae871d87",
            "5cd8a219b69e46aa9913ecf9ac16d8b5",
            "ffcb6442757c4d8fa0b64d6813c261f2",
            "0ea1ef2cd8574f3385da830711032285",
            "bfd298f36bea418b8e80000245c25ed1",
            "2a0634254c42464e960eca2fd4f185a3",
            "9c98cd7ac7e340a2830fe677bf5f60cd",
            "28ea4db6f6d44017b326ce9da7830ca4",
            "288ac80ae11d421bac87e2438e62c2b5",
            "d686c7ce5f32456bb071c49d772678d0",
            "9a8e21f649e440cda780ffca8d67805c",
            "b5eebda55aad4a48889c8e7cf3ab4975",
            "f32cbba9a7b6489a831f9a6bea21938c",
            "d35803ce3faf4011ade3032954d3fc49",
            "f81a4981e95941c3a915b56f9e2f1ea0",
            "2c84144d7ea8458fafb216b32d1d6d3e",
            "13e4f6d9a4be47209385c67158a45bb3",
            "6a6bd6514aa8402e978923dde155551e",
            "a380327b8df24a01b85511c64523ca01",
            "bcc23dfef01e42fea8bb94b0c5e6f3fc",
            "aacad03af7d34da481016a5a2c801bc7",
            "9dd07ae5bb424ebdadf42d9e10196f8f",
            "cf4e654be55140b1ad6d9f5439081ee9",
            "a0ec090f1b154f328f4aa216370ebdce",
            "959de368cabf4d51b46825f895b88b97",
            "45eb295c8b9b413fae926166cd36836c",
            "466f13d171a64be3a5055424d1d71208",
            "a4206c82275847ef8c464265c9eaea08",
            "90eb5c7a2d8f4627bcaae84cfc8f4766",
            "c1521782dbe74f9bac8cda73d2c4f3fb",
            "fe4208bf6680465697a87889ec3f120c",
            "5cfdb87a29b34693b2b65fb915bd4dd6",
            "128df31502d445159c9b82651e01014f",
            "832baac0e0af4bccbc5af062076bcf96",
            "54748003511e4ff49a91c73d3f3df2f6",
            "8bc478ee4f37434a9e92e28a90c659ea",
            "cffa2115c14f40c5be8c0cee67193b49",
            "cb314a81ab75474fb481c52f4a1ed842",
            "e36d5e5b22e0407b810231a97acbf73e",
            "61bd816155a946c9b07a864c22231f85",
            "137927bd797149efb17707da77674f78",
            "41d45ef201984254877e67128611b00f",
            "9652d4067a9344edb35777d73a140e54",
            "44b7a2315a124ed6ae7d01eb9d41e87d",
            "79a8c323bee44560bd087a1f052cb1af",
            "593301961d294c0aa115fe23699fd8e7",
            "2bfb7e0fb6b744b1a1ab281a807e3511",
            "b12b0d90ed1840cda3aa64d36f2f3d7c",
            "599c8f9597aa4d7cbf08fe8740a9841b",
            "acd06f252bca41509d20ca29dd2b8b86",
            "bf459c020258434dbc54abd2761c95fa",
            "ff7a54f4cdcf4c8f9d82df85217c8309",
            "bc9291dba526497fb31cf354685ec8af",
            "8f3ece1cf54b4e7f83c5413bf359e9e3",
            "cc1efb5daac348ba87eb2e52e5c4a776",
            "b364405e003e49afb820eac3827d422d",
            "3d87aaa4f44148839e84d32f7292422e",
            "2e40d6d0905341f295f6e2563741783a",
            "9ac126c24753423698adc9d6c5b48087",
            "3c6ec360930e4a57931cca59cceb136e",
            "ca2557e24b204e9abc13522c5c192e91"
          ]
        }
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading readme:   0%|          | 0.00/7.81k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "eeb454382a57454893131f6521914a73"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/21.0M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d1a81417dae24a53a8b53c47e6b4381c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/20.5M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9c98cd7ac7e340a2830fe677bf5f60cd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/42.0M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6a6bd6514aa8402e978923dde155551e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/25000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "90eb5c7a2d8f4627bcaae84cfc8f4766"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating test split:   0%|          | 0/25000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "61bd816155a946c9b07a864c22231f85"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating unsupervised split:   0%|          | 0/50000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bf459c020258434dbc54abd2761c95fa"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "DatasetDict({\n",
            "    train: Dataset({\n",
            "        features: ['text', 'label'],\n",
            "        num_rows: 25000\n",
            "    })\n",
            "    test: Dataset({\n",
            "        features: ['text', 'label'],\n",
            "        num_rows: 25000\n",
            "    })\n",
            "    unsupervised: Dataset({\n",
            "        features: ['text', 'label'],\n",
            "        num_rows: 50000\n",
            "    })\n",
            "})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dset=dset.shuffle() #This is never a bad idea, datasets may have ordering to them, which is not what we want\n",
        "del dset[\"unsupervised\"] #Delete the unlabeled part of the dataset, we don't need it for anything"
      ],
      "metadata": {
        "id": "lbmgiOj4pWw-"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pprint(dset['train'][0]['text'])\n",
        "print(dset['train'][0]['label'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KI3kyD3vry8C",
        "outputId": "2666b3f2-8982-402a-dc8c-88c438f38d2b"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(\"I would say for it's time, this movie was awesome...and yes if you have no \"\n",
            " 'desire to become a Christian, then why bother watching it. I saw this movie '\n",
            " 'after I had already been saved and found it to be very moving. I see now '\n",
            " 'they have taken these movies to another level and have created the Left '\n",
            " 'Behind series...they run a close comparison and definitely are more modern '\n",
            " 'to reach people. I think in order to actually judge this movie, you should '\n",
            " \"see it,,,there are 3 or 4 of them in the series if I am not mistaken...don't \"\n",
            " 'use our comments to judge, see the movie for yourself!! God will bless you '\n",
            " 'if that is why you are watching them.')\n",
            "1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Tokenize and map vocabulary\n",
        "         \n",
        "*   We need to achieve two complementary tasks\n",
        "*   **Tokenize** split the text into units which can be interpreted as features (words in this case)\n",
        "*   **Map vocabulary** build the feature vector for each example\n",
        "*   Since this is NLP, here it means listing the non-zero elements of the feature vector, or in other words the indices of the vocabulary items\n",
        "* Since we work with the bag of words (BoW) representation, these do not need to be (and are not) in the order in which they appear in the text\n",
        "* These indices then refer to the rows in the embedding matrix\n",
        "*   A traditional and well-tested way it to use sklearn's feature extraction package\n",
        "*   CountVectorizer is most likely what we want in here, because we only want the ids, nothing else\n",
        "* But for other NLP work the TfidfVectorizer is also very handy\n",
        "\n"
      ],
      "metadata": {
        "id": "KMnqQ78cMpk2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import sklearn.feature_extraction\n",
        "\n",
        "# max_features means the size of the vocabulary\n",
        "# which means max_features most-common words\n",
        "vectorizer=sklearn.feature_extraction.text.CountVectorizer(binary=True,max_features=20000)\n",
        "\n",
        "texts=[ex[\"text\"] for ex in dset[\"train\"]] #get a list of all texts from the training data\n",
        "vectorizer.fit(texts) #\"Trains\" the vectorizer, i.e. builds its vocabulary\n"
      ],
      "metadata": {
        "id": "MYOtKJu7Mohd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "outputId": "0a40cefe-fb14-4cc6-e371-a3bc7582c262"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "CountVectorizer(binary=True, max_features=20000)"
            ],
            "text/html": [
              "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>CountVectorizer(binary=True, max_features=20000)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">CountVectorizer</label><div class=\"sk-toggleable__content\"><pre>CountVectorizer(binary=True, max_features=20000)</pre></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Building the feature vectors\n",
        "\n",
        "* This is super-easy with the vectorizer\n",
        "* It produces a sparse matrix of the non-zero elements"
      ],
      "metadata": {
        "id": "UDNy3w8reO6C"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def vectorize_example(ex):\n",
        "    vectorized=vectorizer.transform([ex[\"text\"]]) # [...] because the vectorizer expects a list/iterable over inputs, not one input\n",
        "    non_zero_features=vectorized.nonzero()[1] #.nonzero gives a pair of (rows,columns), we want the columns\n",
        "    non_zero_features+=1 #feature index 0 will have a special meaning\n",
        "                         # so let us not produce it by adding +1 to everything\n",
        "    return {\"input_ids\":non_zero_features}\n",
        "\n",
        "vectorized=vectorize_example(dset[\"train\"][0])"
      ],
      "metadata": {
        "id": "5H9AQOc2rGIO"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(vectorized)"
      ],
      "metadata": {
        "id": "ECqsgmwTXaOC",
        "outputId": "e143c995-d850-44d8-f869-1cca6b2b7a9d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': array([  436,   594,   772,   794,   887,   959,  1115,  1460,  1702,\n",
            "        1743,  1756,  1791,  2052,  2249,  3232,  3429,  3636,  3680,\n",
            "        4255,  4757,  4956,  5429,  7127,  7222,  7762,  8128,  8322,\n",
            "        8929,  9085,  9602,  9630,  9846, 10341, 10409, 11526, 11576,\n",
            "       11681, 11762, 11765, 11766, 12134, 12202, 12237, 12363, 12504,\n",
            "       12515, 12574, 13028, 14312, 15267, 15447, 15455, 15461, 15682,\n",
            "       15790, 16020, 17630, 17893, 17897, 17910, 17917, 17929, 17938,\n",
            "       17944, 17954, 17968, 18074, 18115, 18962, 19112, 19398, 19420,\n",
            "       19609, 19639, 19805, 19912, 19927, 19935], dtype=int32)}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# We can map back to vocabulary and check that everything works\n",
        "# vectorizer.vocabulary_ is a dictionary {key:word, value:idx}\n",
        "\n",
        "idx2word=dict((i,w) for (w,i) in vectorizer.vocabulary_.items()) #inverse the vocab dictionary\n",
        "words=[]\n",
        "for idx in vectorized[\"input_ids\"]:\n",
        "    words.append(idx2word[idx-1]) ## It is easy to forgot we moved all by +1\n",
        "pprint(\", \".join(words)) #This is now the bag of words representation of the document"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yvgd3IrWfCai",
        "outputId": "b80449c1-9a60-4be1-c26e-7c9e2f7db5a4"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "('actually, after, already, am, and, another, are, awesome, be, become, been, '\n",
            " 'behind, bless, bother, christian, close, comments, comparison, created, '\n",
            " 'definitely, desire, don, for, found, god, had, have, if, in, is, it, judge, '\n",
            " 'left, level, mistaken, modern, more, movie, movies, moving, no, not, now, '\n",
            " 'of, or, order, our, people, reach, run, saved, saw, say, see, series, '\n",
            " 'should, taken, that, the, them, then, there, these, they, think, this, time, '\n",
            " 'to, use, very, was, watching, why, will, would, yes, you, yourself')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tokenizing / vectorizing the whole dataset\n",
        "\n",
        "* The datasets library allows us to efficiently map() a function across the whole dataset\n",
        "* Can run in parallel\n",
        "\n",
        "**Note**: confusingly, and unlike the Python`map` function, [`Dataset.map`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map) function _updates_ its argument dataset, keeping existing values. Here, the call adds the values returned by the function call (here `input_ids`) to each example while also keeping the original `text` and `label` values.\n"
      ],
      "metadata": {
        "id": "i33KIMgkiIn7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Apply the tokenizer to the whole dataset using .map()\n",
        "dset_tokenized = dset.map(vectorize_example,num_proc=4)\n",
        "pprint(dset_tokenized[\"train\"][0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "0e858b1ccaef4fd18be0b4079e4c8646",
            "1094731701564aee869d72736d8f9a88",
            "cdc28da3728a49f8915dabf766dc26a8",
            "ccf998160452482fa9d67af801e9d169",
            "c75541bed1794a6bb05890ca783e0bb1",
            "124e307a758c406684bfc6da8997a7d3",
            "2fb1b9bb43d84dd3aacd4589fc2aada4",
            "bb1b8547de064474a0c18bf66cce7889",
            "c3a1eebd59d44f9db9e351eeabc79a9a",
            "d6b3ac69a446496eafc88a80347a5393",
            "9a4b67c0fb774ac7b147098e1efd7260",
            "22941c5f34504afd834c27bc7b14467d",
            "cb72facec89145b5bc20444949ac3200",
            "957fa9d3dc3344faa6b28534b3051e19",
            "e758295d086a495e99fca994d7509ad3",
            "8b1014a383764fe6b5e5c0c341b5e8f5",
            "f622826c23b647c78671cf589f8b389e",
            "b01b1e69045c47f5a743821f14d9d970",
            "15dd150da71c4818ade5f27be603a147",
            "b9b95ed802084d41b4975bf5169ea9ae",
            "7129ce8bc63745ad9605fc60bf086c3a",
            "c2d63cd124644f59a7dabfc69f790948"
          ]
        },
        "id": "33xMYRd0q2B9",
        "outputId": "46afec68-135d-4bef-8373-3e523411e28c"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/25000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0e858b1ccaef4fd18be0b4079e4c8646"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/25000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "22941c5f34504afd834c27bc7b14467d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': [436,\n",
            "               594,\n",
            "               772,\n",
            "               794,\n",
            "               887,\n",
            "               959,\n",
            "               1115,\n",
            "               1460,\n",
            "               1702,\n",
            "               1743,\n",
            "               1756,\n",
            "               1791,\n",
            "               2052,\n",
            "               2249,\n",
            "               3232,\n",
            "               3429,\n",
            "               3636,\n",
            "               3680,\n",
            "               4255,\n",
            "               4757,\n",
            "               4956,\n",
            "               5429,\n",
            "               7127,\n",
            "               7222,\n",
            "               7762,\n",
            "               8128,\n",
            "               8322,\n",
            "               8929,\n",
            "               9085,\n",
            "               9602,\n",
            "               9630,\n",
            "               9846,\n",
            "               10341,\n",
            "               10409,\n",
            "               11526,\n",
            "               11576,\n",
            "               11681,\n",
            "               11762,\n",
            "               11765,\n",
            "               11766,\n",
            "               12134,\n",
            "               12202,\n",
            "               12237,\n",
            "               12363,\n",
            "               12504,\n",
            "               12515,\n",
            "               12574,\n",
            "               13028,\n",
            "               14312,\n",
            "               15267,\n",
            "               15447,\n",
            "               15455,\n",
            "               15461,\n",
            "               15682,\n",
            "               15790,\n",
            "               16020,\n",
            "               17630,\n",
            "               17893,\n",
            "               17897,\n",
            "               17910,\n",
            "               17917,\n",
            "               17929,\n",
            "               17938,\n",
            "               17944,\n",
            "               17954,\n",
            "               17968,\n",
            "               18074,\n",
            "               18115,\n",
            "               18962,\n",
            "               19112,\n",
            "               19398,\n",
            "               19420,\n",
            "               19609,\n",
            "               19639,\n",
            "               19805,\n",
            "               19912,\n",
            "               19927,\n",
            "               19935],\n",
            " 'label': 1,\n",
            " 'text': \"I would say for it's time, this movie was awesome...and yes if you \"\n",
            "         'have no desire to become a Christian, then why bother watching it. I '\n",
            "         'saw this movie after I had already been saved and found it to be '\n",
            "         'very moving. I see now they have taken these movies to another level '\n",
            "         'and have created the Left Behind series...they run a close '\n",
            "         'comparison and definitely are more modern to reach people. I think '\n",
            "         'in order to actually judge this movie, you should see it,,,there are '\n",
            "         \"3 or 4 of them in the series if I am not mistaken...don't use our \"\n",
            "         'comments to judge, see the movie for yourself!! God will bless you '\n",
            "         'if that is why you are watching them.'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Input encoding for MLP\n",
        "\n",
        "* Our `input_ids` are an array containing the indices of the tokens found in the text\n",
        "* This corresponds to the indices into the row of the embedding matrix in the model\n",
        "* That seems to be exactly what we need!\n"
      ],
      "metadata": {
        "id": "XTtyHQpIIJWw"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Batching and padding\n",
        "\n",
        "* When working with neural networks, one rarely trains one example at a time\n",
        "* Instead, processing always happens a batch at a time\n",
        "* This has two important reasons:\n",
        "  1. No batching is too slow (GPU parallelization cannot kick in across examples)\n",
        "  2. The gradients are averaged across the whole batch and applied only once, i.e. batching acts as a regularizer and improves the stability of the training\n"
      ],
      "metadata": {
        "id": "JvaP1DpHjI3f"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Padding and Collation (forming a batch)\n",
        "\n",
        "## Padding:\n",
        "\n",
        "* In order to build a batch as a 2D array of (example, seq), we need to fit together examples of different length\n",
        "* Solution: pad the shorter examples with zeroes to the length of the longest example in the batch\n",
        "* Make sure that zero is understood as padding value rather than a (hypothetical) feature with index 0\n",
        "* This is best shown by example, it is in the end easier than it may sound\n",
        "\n",
        "## Collation:\n",
        "\n",
        "* Much like examples are dictionaries with the data, also batches are dictionaries with the data\n",
        "* The only difference is that in a batch, all data tensors have one extra dimension, that's all there is to it\n",
        "\n",
        "## Collator function:\n",
        "\n",
        "* Padding and collation is taken care of by a single function in the HF libraries\n",
        "* It receives a list of examples, and returns a ready batch\n",
        "* The surrounding library code takes care of forming these lists\n",
        "* Let's try to implement one below"
      ],
      "metadata": {
        "id": "hNYxw92nj51B"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# 1) I need to define it here, will explain below\n",
        "# 2) I show here a very straightforward implementation of padding and collation\n",
        "# 3) Normally, one would use transformers.DataCollatorWithPadding but that assumes\n",
        "#    a particular tokenizer, to which it outsources much of the work, and we do not\n",
        "#    have it\n",
        "def collator(list_of_examples):\n",
        "    #this is easy, labels are made into a single tensor\n",
        "    batch={\"labels\":torch.tensor(list(ex[\"label\"] for ex in list_of_examples))}\n",
        "    #the worse bit is now to pad the examples, as they are of different length\n",
        "    tensors=[]\n",
        "    max_len=max(len(example[\"input_ids\"]) for example in list_of_examples) #this is the longest example in the batch\n",
        "    #everything needs to be padded to fit in length the longest example\n",
        "    #(so we can build a single tensor out of it)\n",
        "    for example in list_of_examples:\n",
        "        ids=torch.tensor(example[\"input_ids\"]) #pick the input ids\n",
        "        # pad(what,(from_left, from_right)) <- this is how we call the stock pad function\n",
        "        padded=torch.nn.functional.pad(ids,(0,max_len-ids.shape[0])) #pad by max - current length, pads with zero by default\n",
        "        tensors.append(padded) #accumulated the padded ids\n",
        "    batch[\"input_ids\"]=torch.vstack(tensors) #now that we have all of them the same length, a simple vstack() stacks them up\n",
        "    return batch #...and that's all there is to it\n",
        "\n",
        "\n",
        "\n",
        "#Build a batch from 2 examples, with padding\n",
        "batch=collator([dset_tokenized[\"train\"][2],dset_tokenized[\"train\"][7]])\n",
        "print(\"Shape of labels:\",batch[\"labels\"].shape)\n",
        "print(\"Shape of input_ids:\",batch[\"input_ids\"].shape)\n",
        "pprint(batch[\"labels\"])\n",
        "pprint(batch[\"input_ids\"])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zXXDxNo6kwfA",
        "outputId": "f09fa880-47c2-42d9-b137-f21aa2a78f4b"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape of labels: torch.Size([2])\n",
            "Shape of input_ids: torch.Size([2, 87])\n",
            "tensor([0, 1])\n",
            "tensor([[   10,   419,   430,   887,   959,  1039,  1300,  1435,  1517,  1674,\n",
            "          1702,  1897,  2115,  2604,  2625,  2691,  2790,  4130,  4292,  4620,\n",
            "          5104,  5403,  5434,  5838,  6270,  6311,  6783,  7346,  7775,  7801,\n",
            "          8322,  8378,  8731,  9085,  9602,  9630, 10395, 10461, 10475, 10893,\n",
            "         11656, 11681, 11762, 11765, 11993, 12247, 12363, 12437, 12465, 12577,\n",
            "         13028, 13091, 13292, 13482, 13783, 13833, 14277, 15070, 15502, 15505,\n",
            "         15682, 15696, 16033, 16041, 16508, 16603, 16664, 17075, 17189, 17513,\n",
            "         17885, 17893, 17897, 17917, 17929, 17968, 18018, 18081, 18115, 18175,\n",
            "         19273, 19308, 19521, 19712, 19800, 19805, 19918],\n",
            "        [  727,   806,   860,   927,  2604,  2625,  4045,  5163,  5395,  5711,\n",
            "          5869,  6554,  6878,  6894,  7127,  7744,  7762,  7940,  8186,  8331,\n",
            "          8350,  8454,  8710,  9085,  9602,  9630,  9777, 10398, 10475, 10829,\n",
            "         11605, 11832, 12363, 13470, 13800, 13804, 14691, 14968, 15589, 16033,\n",
            "         16478, 16542, 16544, 17018, 17075, 17897, 17968, 18730, 19112, 19120,\n",
            "         19398, 19420, 19614, 19712, 19801,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Build the MLP model\n",
        "\n",
        "* Now that all of our data is in shape, we can build the model\n",
        "* That is luckily quite easy in this case\n",
        "\n",
        "The model class in its simplest form has `__init__()` which instantiates the layers and `forward()` which implements the actual computation. For more information on these, please see the [PyTorch turorial](https://pytorch.org/tutorials/beginner/introyt/modelsyt_tutorial.html)."
      ],
      "metadata": {
        "id": "AOYYF5I1OWG9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import transformers\n",
        "\n",
        "# A model wants a config, I can simply inherit from the base\n",
        "# class for pretrained configs\n",
        "class MLPConfig(transformers.PretrainedConfig):\n",
        "    pass\n",
        "\n",
        "# This is the model\n",
        "class MLP(transformers.PreTrainedModel):\n",
        "\n",
        "    config_class=MLPConfig\n",
        "\n",
        "    # In the initialization method, one instantiates the layers\n",
        "    # these will be, for the most part the trained parameters of the model\n",
        "    def __init__(self,config):\n",
        "        super().__init__(config)\n",
        "        self.vocab_size=config.vocab_size #embedding matrix row count\n",
        "        # Build and initialize embedding of vocab size +1 x hidden size (+1 because of the padding index 0!)\n",
        "        self.embedding=torch.nn.Embedding(num_embeddings=self.vocab_size+1,embedding_dim=config.hidden_size,padding_idx=0)\n",
        "        # Normally you would not initialize these yourself, but I have my reasons here ;)\n",
        "        torch.nn.init.uniform_(self.embedding.weight.data,-0.001,0.001) #initialize the embeddings with small random values\n",
        "        # Note! This function is relatively clever and keeps the embedding for 0, the padding, pure zeros\n",
        "        # This takes care of the lower half of the network, now the upper half\n",
        "        # Output layer: hidden size x output size\n",
        "        self.output=torch.nn.Linear(in_features=config.hidden_size,out_features=config.nlabels)\n",
        "        # Now we have the parameters of the model\n",
        "\n",
        "\n",
        "    # The computation of the model is put into the forward() function\n",
        "    # it receives a batch of data and optionally the correct `labels`\n",
        "    #\n",
        "    # If given `labels` it returns (loss,output)\n",
        "    # if not, then it returns (output,)\n",
        "    def forward(self,input_ids,labels=None):\n",
        "        #1) sum up the embeddings of the items\n",
        "        embedded=self.embedding(input_ids) #(batch,ids)->(batch,ids,embedding_dim)\n",
        "        # Since the Embedding keeps the first row of the matrix pure zeros, we don't need to worry about the padding\n",
        "        # so next we sum the embeddings across the word dimension\n",
        "        # (batch,ids,embedding_dim) -> (batch,embedding_dim)\n",
        "        embedded_summed=torch.sum(embedded,dim=1)\n",
        "\n",
        "        #2) apply non-linearity\n",
        "        # (batch,embedding_dim) -> (batch,embedding_dim)\n",
        "        projected=torch.tanh(embedded_summed) #Note how non-linearity is applied here and not when configuring the layer in __init__()\n",
        "\n",
        "        #3) and now apply the upper, output layer of the network\n",
        "        # (batch,embedding_dim) -> (batch, num_of_classes i.e. 2 in our case)\n",
        "        logits=self.output(projected)\n",
        "\n",
        "        # ...and that's all there is to it!\n",
        "\n",
        "        #print(\"input_ids.shape\",input_ids.shape)\n",
        "        #print(\"embedded.shape\",embedded.shape)\n",
        "        #print(\"embedded_summed.shape\",embedded_summed.shape)\n",
        "        #print(\"projected.shape\",projected.shape)\n",
        "        #print(\"logits.shape\",logits.shape)\n",
        "\n",
        "        # If we have labels, we ought to calculate the loss\n",
        "        if labels is not None:\n",
        "            loss=torch.nn.CrossEntropyLoss() #This loss is meant for classification, so let's use it\n",
        "            # You run it as loss(model_output,correct_labels)\n",
        "            return (loss(logits,labels),logits)\n",
        "        else:\n",
        "            # No labels, so just return the logits\n",
        "            return (logits,)\n",
        "\n"
      ],
      "metadata": {
        "id": "vP4FtrMwCpGi"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Configure the model:\n",
        "#   these parameters are used in the model's __init__()\n",
        "mlp_config=MLPConfig(vocab_size=len(vectorizer.vocabulary_),hidden_size=20,nlabels=2)\n",
        "\n",
        "# And now we can instantiate it\n",
        "mlp=MLP(mlp_config)\n",
        "\n",
        "#we can make a little test with a fake batch formed by the two first example\n",
        "fake_batch=collator([dset_tokenized[\"train\"][0],dset_tokenized[\"train\"][1]])\n",
        "mlp(**fake_batch) #** expands input_ids and labels as parameters of the call"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JQjSGUYGwrzh",
        "outputId": "dee23720-0d1b-4037-fca2-8906e3ae7740"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(0.6549, grad_fn=<NllLossBackward0>),\n",
              " tensor([[0.1117, 0.1910],\n",
              "         [0.1164, 0.1932]], grad_fn=<AddmmBackward0>))"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Train the model\n",
        "\n",
        "We will use the Hugging Face [Trainer](https://huggingface.co/docs/transformers/main_classes/trainer) class for training\n",
        "\n",
        "* Loads of arguments that control the training\n",
        "* Configurable metrics to evaluate performance\n",
        "* Data collator builds the batches\n",
        "* Early stopping callback stops when eval loss no longer improves\n",
        "* Model load/save\n",
        "* Excellent foundation for later deep learning course\n",
        "  "
      ],
      "metadata": {
        "id": "tdlcMObzQGGP"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "First, let's create a [`TrainingArguments`](https://huggingface.co/docs/transformers/v4.17.0/en/main_classes/trainer#transformers.TrainingArguments) object to specify hyperparameters and various other settings for training.\n",
        "\n",
        "Printing this simple dataclass object will show not only the values we set, but also the defaults for all other arguments. Don't worry if you don't understand what all of these do! Many are not relevant to us here, and you can find the details in [`Trainer` documentation](https://huggingface.co/docs/transformers/main_classes/trainer) if you are interested."
      ],
      "metadata": {
        "id": "aZgcNi4B76SX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Set training arguments\n",
        "# their names are mostly self-explanatory\n",
        "trainer_args = transformers.TrainingArguments(\n",
        "    \"mlp_checkpoints\", #save checkpoints here\n",
        "    evaluation_strategy=\"steps\",\n",
        "    logging_strategy=\"steps\",\n",
        "    eval_steps=500,\n",
        "    logging_steps=500,\n",
        "    learning_rate=1e-5, #learning rate of the gradient descent\n",
        "    max_steps=20000,\n",
        "    load_best_model_at_end=True,\n",
        "    per_device_train_batch_size=128\n",
        ")\n",
        "\n",
        "pprint(trainer_args)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IhfdW62z8cCn",
        "outputId": "2d40ac6c-eec5-490a-b4ff-03b6a8334fc7"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "TrainingArguments(\n",
            "_n_gpu=1,\n",
            "accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True},\n",
            "adafactor=False,\n",
            "adam_beta1=0.9,\n",
            "adam_beta2=0.999,\n",
            "adam_epsilon=1e-08,\n",
            "auto_find_batch_size=False,\n",
            "bf16=False,\n",
            "bf16_full_eval=False,\n",
            "data_seed=None,\n",
            "dataloader_drop_last=False,\n",
            "dataloader_num_workers=0,\n",
            "dataloader_persistent_workers=False,\n",
            "dataloader_pin_memory=True,\n",
            "dataloader_prefetch_factor=None,\n",
            "ddp_backend=None,\n",
            "ddp_broadcast_buffers=None,\n",
            "ddp_bucket_cap_mb=None,\n",
            "ddp_find_unused_parameters=None,\n",
            "ddp_timeout=1800,\n",
            "debug=[],\n",
            "deepspeed=None,\n",
            "disable_tqdm=False,\n",
            "dispatch_batches=None,\n",
            "do_eval=True,\n",
            "do_predict=False,\n",
            "do_train=False,\n",
            "eval_accumulation_steps=None,\n",
            "eval_delay=0,\n",
            "eval_steps=500,\n",
            "evaluation_strategy=steps,\n",
            "fp16=False,\n",
            "fp16_backend=auto,\n",
            "fp16_full_eval=False,\n",
            "fp16_opt_level=O1,\n",
            "fsdp=[],\n",
            "fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},\n",
            "fsdp_min_num_params=0,\n",
            "fsdp_transformer_layer_cls_to_wrap=None,\n",
            "full_determinism=False,\n",
            "gradient_accumulation_steps=1,\n",
            "gradient_checkpointing=False,\n",
            "gradient_checkpointing_kwargs=None,\n",
            "greater_is_better=False,\n",
            "group_by_length=False,\n",
            "half_precision_backend=auto,\n",
            "hub_always_push=False,\n",
            "hub_model_id=None,\n",
            "hub_private_repo=False,\n",
            "hub_strategy=every_save,\n",
            "hub_token=<HUB_TOKEN>,\n",
            "ignore_data_skip=False,\n",
            "include_inputs_for_metrics=False,\n",
            "include_num_input_tokens_seen=False,\n",
            "include_tokens_per_second=False,\n",
            "jit_mode_eval=False,\n",
            "label_names=None,\n",
            "label_smoothing_factor=0.0,\n",
            "learning_rate=1e-05,\n",
            "length_column_name=length,\n",
            "load_best_model_at_end=True,\n",
            "local_rank=0,\n",
            "log_level=passive,\n",
            "log_level_replica=warning,\n",
            "log_on_each_node=True,\n",
            "logging_dir=mlp_checkpoints/runs/Mar25_13-19-07_a577afc2cb54,\n",
            "logging_first_step=False,\n",
            "logging_nan_inf_filter=True,\n",
            "logging_steps=500,\n",
            "logging_strategy=steps,\n",
            "lr_scheduler_kwargs={},\n",
            "lr_scheduler_type=linear,\n",
            "max_grad_norm=1.0,\n",
            "max_steps=20000,\n",
            "metric_for_best_model=loss,\n",
            "mp_parameters=,\n",
            "neftune_noise_alpha=None,\n",
            "no_cuda=False,\n",
            "num_train_epochs=3.0,\n",
            "optim=adamw_torch,\n",
            "optim_args=None,\n",
            "output_dir=mlp_checkpoints,\n",
            "overwrite_output_dir=False,\n",
            "past_index=-1,\n",
            "per_device_eval_batch_size=8,\n",
            "per_device_train_batch_size=128,\n",
            "prediction_loss_only=False,\n",
            "push_to_hub=False,\n",
            "push_to_hub_model_id=None,\n",
            "push_to_hub_organization=None,\n",
            "push_to_hub_token=<PUSH_TO_HUB_TOKEN>,\n",
            "ray_scope=last,\n",
            "remove_unused_columns=True,\n",
            "report_to=['tensorboard'],\n",
            "resume_from_checkpoint=None,\n",
            "run_name=mlp_checkpoints,\n",
            "save_on_each_node=False,\n",
            "save_only_model=False,\n",
            "save_safetensors=True,\n",
            "save_steps=500,\n",
            "save_strategy=steps,\n",
            "save_total_limit=None,\n",
            "seed=42,\n",
            "skip_memory_metrics=True,\n",
            "split_batches=None,\n",
            "tf32=None,\n",
            "torch_compile=False,\n",
            "torch_compile_backend=None,\n",
            "torch_compile_mode=None,\n",
            "torchdynamo=None,\n",
            "tpu_metrics_debug=False,\n",
            "tpu_num_cores=None,\n",
            "use_cpu=False,\n",
            "use_ipex=False,\n",
            "use_legacy_prediction_loop=False,\n",
            "use_mps_device=False,\n",
            "warmup_ratio=0.0,\n",
            "warmup_steps=0,\n",
            "weight_decay=0.0,\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Next, let's create a metric for evaluating performance during and after training. We can use the convenience function [`load_metric`](https://huggingface.co/docs/datasets/about_metrics) to load one of many pre-made metrics and wrap this for use by the trainer.\n",
        "\n",
        "As the task is simple binary classification and our data is even 50:50 balanced, we can comfortably use the basic `accuracy` metric, defined as the proportion of correctly predicted labels out of all labels."
      ],
      "metadata": {
        "id": "4sJwNXPU-dQX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import evaluate\n",
        "\n",
        "accuracy = evaluate.load(\"accuracy\")\n",
        "\n",
        "def compute_accuracy(outputs_and_labels):\n",
        "    outputs, labels = outputs_and_labels\n",
        "    predictions = np.argmax(outputs, axis=-1) #pick the index of the \"winning\" label\n",
        "    return accuracy.compute(predictions=predictions, references=labels)"
      ],
      "metadata": {
        "id": "u3jxIItb0BL9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "c29e659569f141229de8e4a25ccc00f3",
            "01d2e73d04914715b58c7b2227e5defe",
            "6b8ec5013a5b4fbabfe2a9a63777a3e5",
            "1445fa6563764220a29cbd7d30045d2c",
            "bcf5048367eb46bd97f800049a0a164c",
            "3bb2265a872a40cb9704dee3cf5e8858",
            "189c4e0f87a64e81a62f92ef431d133b",
            "9b8c5af3524146e2ad367d7b2ba65486",
            "ba26f013e85341eca873279c5dafbdf3",
            "2acbffae3c3f44428c8cdc494004cd5e",
            "ccf5918a328f44da9a5d061b46cb89ef"
          ]
        },
        "outputId": "194d58a9-2a06-40dc-f864-d2ee21fa8761"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/4.20k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c29e659569f141229de8e4a25ccc00f3"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can then create the `Trainer` and train the model by invoking the [`Trainer.train`](https://huggingface.co/docs/transformers/main_classes/trainer#transformers.Trainer.train) function.\n",
        "\n",
        "In addition to the model, the settings passed in through the `TrainingArguments` object created above (`trainer_args`), the data, and the metric defined above, we create and pass the following to the `Trainer`:\n",
        "\n",
        "* [data collator](https://huggingface.co/docs/transformers/main_classes/data_collator): groups input into batches\n",
        "* [`EarlyStoppingCallback`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.EarlyStoppingCallback): stops training when performance stops improving"
      ],
      "metadata": {
        "id": "S7kbz8uU-zpQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Make a new model\n",
        "mlp = MLP(mlp_config)\n",
        "\n",
        "\n",
        "# Argument gives the number of steps of patience before early stopping\n",
        "# i.e. training is stopped when the evaluation loss fails to improve\n",
        "# certain number of times\n",
        "early_stopping = transformers.EarlyStoppingCallback(5)\n",
        "\n",
        "trainer = transformers.Trainer(\n",
        "    model=mlp,\n",
        "    args=trainer_args,\n",
        "    train_dataset=dset_tokenized[\"train\"],\n",
        "    eval_dataset=dset_tokenized[\"test\"].select(range(1000)), #make a smaller subset to evaluate on\n",
        "    compute_metrics=compute_accuracy,\n",
        "    data_collator=collator,\n",
        "    callbacks=[early_stopping]\n",
        ")\n",
        "\n",
        "# FINALLY!\n",
        "trainer.train()"
      ],
      "metadata": {
        "id": "AoEoWsj4P_zN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "cad78b34-185a-4911-cf21-2c207f86f922"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:432: FutureWarning: Passing the following arguments to `Accelerator` is deprecated and will be removed in version 1.0 of Accelerate: dict_keys(['dispatch_batches', 'split_batches', 'even_batches', 'use_seedable_sampler']). Please pass an `accelerate.DataLoaderConfiguration` instead: \n",
            "dataloader_config = DataLoaderConfiguration(dispatch_batches=None, split_batches=False, even_batches=True, use_seedable_sampler=True)\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='20000' max='20000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [20000/20000 06:26, Epoch 102/103]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.666600</td>\n",
              "      <td>0.646069</td>\n",
              "      <td>0.756000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.613800</td>\n",
              "      <td>0.601771</td>\n",
              "      <td>0.814000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.566400</td>\n",
              "      <td>0.562668</td>\n",
              "      <td>0.829000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.525400</td>\n",
              "      <td>0.529494</td>\n",
              "      <td>0.848000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>0.489800</td>\n",
              "      <td>0.501324</td>\n",
              "      <td>0.852000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>0.459900</td>\n",
              "      <td>0.477174</td>\n",
              "      <td>0.858000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3500</td>\n",
              "      <td>0.433300</td>\n",
              "      <td>0.456486</td>\n",
              "      <td>0.862000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4000</td>\n",
              "      <td>0.410900</td>\n",
              "      <td>0.438788</td>\n",
              "      <td>0.867000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4500</td>\n",
              "      <td>0.390500</td>\n",
              "      <td>0.423057</td>\n",
              "      <td>0.869000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5000</td>\n",
              "      <td>0.372500</td>\n",
              "      <td>0.409385</td>\n",
              "      <td>0.868000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5500</td>\n",
              "      <td>0.358100</td>\n",
              "      <td>0.397563</td>\n",
              "      <td>0.868000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6000</td>\n",
              "      <td>0.343900</td>\n",
              "      <td>0.387625</td>\n",
              "      <td>0.869000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6500</td>\n",
              "      <td>0.331000</td>\n",
              "      <td>0.378757</td>\n",
              "      <td>0.872000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7000</td>\n",
              "      <td>0.321300</td>\n",
              "      <td>0.370970</td>\n",
              "      <td>0.873000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7500</td>\n",
              "      <td>0.310000</td>\n",
              "      <td>0.364157</td>\n",
              "      <td>0.872000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8000</td>\n",
              "      <td>0.301500</td>\n",
              "      <td>0.357776</td>\n",
              "      <td>0.874000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8500</td>\n",
              "      <td>0.293500</td>\n",
              "      <td>0.351931</td>\n",
              "      <td>0.875000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9000</td>\n",
              "      <td>0.285500</td>\n",
              "      <td>0.347066</td>\n",
              "      <td>0.875000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9500</td>\n",
              "      <td>0.279500</td>\n",
              "      <td>0.342933</td>\n",
              "      <td>0.876000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10000</td>\n",
              "      <td>0.273000</td>\n",
              "      <td>0.339379</td>\n",
              "      <td>0.873000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10500</td>\n",
              "      <td>0.267500</td>\n",
              "      <td>0.335880</td>\n",
              "      <td>0.872000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11000</td>\n",
              "      <td>0.262600</td>\n",
              "      <td>0.332647</td>\n",
              "      <td>0.874000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11500</td>\n",
              "      <td>0.257600</td>\n",
              "      <td>0.330293</td>\n",
              "      <td>0.873000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12000</td>\n",
              "      <td>0.253100</td>\n",
              "      <td>0.327895</td>\n",
              "      <td>0.874000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12500</td>\n",
              "      <td>0.249400</td>\n",
              "      <td>0.325597</td>\n",
              "      <td>0.877000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13000</td>\n",
              "      <td>0.246100</td>\n",
              "      <td>0.323907</td>\n",
              "      <td>0.875000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13500</td>\n",
              "      <td>0.242700</td>\n",
              "      <td>0.322304</td>\n",
              "      <td>0.876000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14000</td>\n",
              "      <td>0.239600</td>\n",
              "      <td>0.321025</td>\n",
              "      <td>0.878000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14500</td>\n",
              "      <td>0.237400</td>\n",
              "      <td>0.319521</td>\n",
              "      <td>0.878000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15000</td>\n",
              "      <td>0.235500</td>\n",
              "      <td>0.318706</td>\n",
              "      <td>0.878000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15500</td>\n",
              "      <td>0.231800</td>\n",
              "      <td>0.317466</td>\n",
              "      <td>0.877000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16000</td>\n",
              "      <td>0.230700</td>\n",
              "      <td>0.316482</td>\n",
              "      <td>0.878000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16500</td>\n",
              "      <td>0.228800</td>\n",
              "      <td>0.316023</td>\n",
              "      <td>0.877000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17000</td>\n",
              "      <td>0.228400</td>\n",
              "      <td>0.315161</td>\n",
              "      <td>0.878000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17500</td>\n",
              "      <td>0.226200</td>\n",
              "      <td>0.314700</td>\n",
              "      <td>0.878000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18000</td>\n",
              "      <td>0.225300</td>\n",
              "      <td>0.314382</td>\n",
              "      <td>0.876000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18500</td>\n",
              "      <td>0.224900</td>\n",
              "      <td>0.314068</td>\n",
              "      <td>0.876000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19000</td>\n",
              "      <td>0.223500</td>\n",
              "      <td>0.313811</td>\n",
              "      <td>0.877000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19500</td>\n",
              "      <td>0.224100</td>\n",
              "      <td>0.313708</td>\n",
              "      <td>0.878000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20000</td>\n",
              "      <td>0.223300</td>\n",
              "      <td>0.313660</td>\n",
              "      <td>0.878000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Checkpoint destination directory mlp_checkpoints/checkpoint-500 already exists and is non-empty. Saving will proceed but saved results may be invalid.\n",
            "Checkpoint destination directory mlp_checkpoints/checkpoint-1000 already exists and is non-empty. Saving will proceed but saved results may be invalid.\n",
            "Checkpoint destination directory mlp_checkpoints/checkpoint-1500 already exists and is non-empty. Saving will proceed but saved results may be invalid.\n",
            "Checkpoint destination directory mlp_checkpoints/checkpoint-2000 already exists and is non-empty. Saving will proceed but saved results may be invalid.\n",
            "Checkpoint destination directory mlp_checkpoints/checkpoint-2500 already exists and is non-empty. Saving will proceed but saved results may be invalid.\n",
            "Checkpoint destination directory mlp_checkpoints/checkpoint-3000 already exists and is non-empty. Saving will proceed but saved results may be invalid.\n",
            "Checkpoint destination directory mlp_checkpoints/checkpoint-3500 already exists and is non-empty. Saving will proceed but saved results may be invalid.\n",
            "Checkpoint destination directory mlp_checkpoints/checkpoint-4000 already exists and is non-empty. Saving will proceed but saved results may be invalid.\n",
            "Checkpoint destination directory mlp_checkpoints/checkpoint-4500 already exists and is non-empty. Saving will proceed but saved results may be invalid.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=20000, training_loss=0.3196216899871826, metrics={'train_runtime': 386.2499, 'train_samples_per_second': 6627.834, 'train_steps_per_second': 51.78, 'total_flos': 274638131712.0, 'train_loss': 0.3196216899871826, 'epoch': 102.04})"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can then evaluate the trained model on a given dataset (here our test subset) by calling [`Trainer.evaluate`](https://huggingface.co/docs/transformers/main_classes/trainer#transformers.Trainer.evaluate):"
      ],
      "metadata": {
        "id": "Td03fcIa-6Hj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "eval_results = trainer.evaluate(dset_tokenized[\"test\"])\n",
        "\n",
        "print(eval_results)"
      ],
      "metadata": {
        "id": "9nlEwpnF2Vow",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "outputId": "6c8ef554-260e-4b15-99ae-deea8c11b1a9"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3125/3125 00:06]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'eval_loss': 0.30097299814224243, 'eval_accuracy': 0.88492, 'eval_runtime': 6.4666, 'eval_samples_per_second': 3866.027, 'eval_steps_per_second': 483.253, 'epoch': 102.04}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Save the model for later use\n",
        "\n",
        "* You can save it with `trainer.save_model()`\n",
        "* You can load it with `MLP.from_pretrained()`\n"
      ],
      "metadata": {
        "id": "P0OYB3TRp-IH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.save_model(\"mlp-imdb\")"
      ],
      "metadata": {
        "id": "FosHTDw3p9dd"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Check save/load"
      ],
      "metadata": {
        "id": "8D8W9EN-zMAc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "mlp2=MLP.from_pretrained(\"mlp-imdb\")"
      ],
      "metadata": {
        "id": "UKMMNbABqLTg"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainer = transformers.Trainer(\n",
        "    model=mlp2,\n",
        "    args=trainer_args,\n",
        "    train_dataset=dset_tokenized[\"train\"],\n",
        "    eval_dataset=dset_tokenized[\"test\"],\n",
        "    compute_metrics=compute_accuracy,\n",
        "    data_collator=collator,\n",
        "    callbacks=[early_stopping]\n",
        ")"
      ],
      "metadata": {
        "id": "pfLnUENrrCyp",
        "outputId": "bb2c97d9-10c0-42b3-b090-dd9a69499309",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:432: FutureWarning: Passing the following arguments to `Accelerator` is deprecated and will be removed in version 1.0 of Accelerate: dict_keys(['dispatch_batches', 'split_batches', 'even_batches', 'use_seedable_sampler']). Please pass an `accelerate.DataLoaderConfiguration` instead: \n",
            "dataloader_config = DataLoaderConfiguration(dispatch_batches=None, split_batches=False, even_batches=True, use_seedable_sampler=True)\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "eval_results = trainer.evaluate(dset_tokenized[\"test\"])\n",
        "print(eval_results)\n",
        "print('Accuracy:', eval_results['eval_accuracy'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 92
        },
        "id": "hpOM2ZwErMGf",
        "outputId": "4b32b8ba-8c72-430b-cfff-22f99be57a2e"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3125/3125 00:11]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'eval_loss': 0.30097299814224243, 'eval_accuracy': 0.88492, 'eval_runtime': 11.5689, 'eval_samples_per_second': 2160.958, 'eval_steps_per_second': 270.12}\n",
            "Accuracy: 0.88492\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Extra time left?\n",
        "\n",
        "* Read through the TrainingArguments documentation, try to understand at least some parts of it https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments\n",
        "* Read through Torch tensor operations, try to understand at least some parts of it: https://pytorch.org/docs/stable/tensors.html\n",
        "* Run the model with different parameters (hidden layer width, learning rate, etc), how much do the results change?\n"
      ],
      "metadata": {
        "id": "8UT5MV1LtSBh"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# What has the model learned?\n",
        "\n",
        "* The embeddings should have some meaning to them\n",
        "* Similar features should have similar embeddings"
      ],
      "metadata": {
        "id": "13aB7DuqzeFi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Grab the embedding matrix out of the trained model\n",
        "# and drop the first row (padding 0)\n",
        "# then we can treat the embeddings as vectors\n",
        "# and maybe compare them to each other\n",
        "# ha ha this below took some googling\n",
        "weights=mlp.embedding.weight.detach().cpu().numpy()\n",
        "weights=weights[1:,:]"
      ],
      "metadata": {
        "id": "M6TUrVkMCmz7"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "qry_idx=vectorizer.vocabulary_[\"great\"] #embedding of \"great\"\n",
        "\n",
        "#calculate the distance of the \"lousy\" embedding to all other embeddings\n",
        "distance_to_qry=sklearn.metrics.pairwise.euclidean_distances(weights[qry_idx:qry_idx+1,:],weights)\n",
        "nearest_neighbors=np.argsort(distance_to_qry) #indices of words nearest to \"lousy\"\n",
        "for nearest in nearest_neighbors[0,:20]:\n",
        "    print(idx2word[nearest])\n",
        "# This works great!"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4uTTvRgGDU9D",
        "outputId": "096f46dc-ed8e-433b-f77b-a6169fdb424b"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "great\n",
            "perfect\n",
            "excellent\n",
            "amazing\n",
            "wonderful\n",
            "loved\n",
            "superb\n",
            "favorite\n",
            "fantastic\n",
            "best\n",
            "wonderfully\n",
            "gem\n",
            "highly\n",
            "today\n",
            "enjoyed\n",
            "refreshing\n",
            "fun\n",
            "rare\n",
            "incredible\n",
            "perfectly\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "* The embeddings indeed seem to reflect the task\n",
        "* There is a meaning to them"
      ],
      "metadata": {
        "id": "cQCmThdu2LDR"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Feature weights\n",
        "\n",
        "*   A typical \"old-school\" way to approach the classification would be a simple linear model, like LinearSVM\n",
        "*   Under such model, each feature (word) would have a single one weight\n",
        "*   And the classification would simply be based on the sum of these weights\n",
        "*   In this context of this task, \"positive\" words would get a high weight, \"negative\" words would get a low weight\n",
        "*   It is in fact quite easy to reconfigure the MLP model to work more or less like this and this effect can be replicated\n",
        "*   I will leave that as an exercise for you\n",
        "\n"
      ],
      "metadata": {
        "id": "Jydy3ECK3O2w"
      }
    }
  ]
}