DPO_Training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "V100"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "5b688adca60f44588e4a3d564422911d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_28bab1e70714420cb57a5fb8109d9031",
              "IPY_MODEL_574454f2f9974f04a0c7ee0f697e5df4",
              "IPY_MODEL_b7ad4e45448849f99b4ab121a58e16b0"
            ],
            "layout": "IPY_MODEL_0b210649cab24fcd84668179a8ba17b2"
          }
        },
        "28bab1e70714420cb57a5fb8109d9031": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_df165e8954494419a09ad57cdc8afa0b",
            "placeholder": "​",
            "style": "IPY_MODEL_d0c67c0da65c42e380bd5670aa99be0b",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "574454f2f9974f04a0c7ee0f697e5df4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ecb6c568610040368f4be9156851c8a5",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_50d2c3a5e5764c828f03a08e4465c6dd",
            "value": 2
          }
        },
        "b7ad4e45448849f99b4ab121a58e16b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_485a2162bacb4047b2362ff5757414be",
            "placeholder": "​",
            "style": "IPY_MODEL_876ac1c941cb44028e55246f6423cc20",
            "value": " 2/2 [00:09&lt;00:00,  4.58s/it]"
          }
        },
        "0b210649cab24fcd84668179a8ba17b2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "df165e8954494419a09ad57cdc8afa0b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d0c67c0da65c42e380bd5670aa99be0b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ecb6c568610040368f4be9156851c8a5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "50d2c3a5e5764c828f03a08e4465c6dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "485a2162bacb4047b2362ff5757414be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "876ac1c941cb44028e55246f6423cc20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "870e76e22db24d87a915ec1892e13d9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0f0b8406b6064d4e8214c8c6bfe3c8f7",
              "IPY_MODEL_7ec4e356994e49718507b3ad05f9ecef",
              "IPY_MODEL_e90285eeeb4547ada23ea7b77d017485"
            ],
            "layout": "IPY_MODEL_3437b1214fb1460c92c858acf5eb0106"
          }
        },
        "0f0b8406b6064d4e8214c8c6bfe3c8f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_57a6625d1dda4bd19c738fddd1890fd2",
            "placeholder": "​",
            "style": "IPY_MODEL_c3b58d9680ab4d82874ab61b1b82a4e6",
            "value": "Map: 100%"
          }
        },
        "7ec4e356994e49718507b3ad05f9ecef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_914cb3d8c1944f87a5d532de19766fa5",
            "max": 507,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c8dffcfd0f5046a486324bcfda1045d1",
            "value": 507
          }
        },
        "e90285eeeb4547ada23ea7b77d017485": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fd101a946184452fbac090de11a5875d",
            "placeholder": "​",
            "style": "IPY_MODEL_fd8fb341e7174e4bb5b172c867f62fd3",
            "value": " 507/507 [00:02&lt;00:00, 245.89 examples/s]"
          }
        },
        "3437b1214fb1460c92c858acf5eb0106": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "57a6625d1dda4bd19c738fddd1890fd2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c3b58d9680ab4d82874ab61b1b82a4e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "914cb3d8c1944f87a5d532de19766fa5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c8dffcfd0f5046a486324bcfda1045d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fd101a946184452fbac090de11a5875d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fd8fb341e7174e4bb5b172c867f62fd3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2a45f4c99dc04903bb06dd4e1a8c5f26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0e3bdb262a904040a08b803241eed129",
              "IPY_MODEL_d29fae3058994ce5a02bf93c7c6b5f26",
              "IPY_MODEL_347056a5202f48deb349bef84163cb95"
            ],
            "layout": "IPY_MODEL_ff46e0588bc04c23a66ce0cd3a459c0f"
          }
        },
        "0e3bdb262a904040a08b803241eed129": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_90a0bf1c376f487a971abe06725e46fc",
            "placeholder": "​",
            "style": "IPY_MODEL_f0285810217f449cbcda96ad2a0eb711",
            "value": "Generating train split: "
          }
        },
        "d29fae3058994ce5a02bf93c7c6b5f26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f66ed8eb22b644bf91341b5447eaaaa9",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_754b5a8bf9b04b2d94e60f870a356da2",
            "value": 1
          }
        },
        "347056a5202f48deb349bef84163cb95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_90b01f27139e40dc927fdf6e5aa342e1",
            "placeholder": "​",
            "style": "IPY_MODEL_ae0c58cbba624797a443905563e080f7",
            "value": " 116722/0 [00:01&lt;00:00, 62104.39 examples/s]"
          }
        },
        "ff46e0588bc04c23a66ce0cd3a459c0f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "90a0bf1c376f487a971abe06725e46fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f0285810217f449cbcda96ad2a0eb711": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f66ed8eb22b644bf91341b5447eaaaa9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "754b5a8bf9b04b2d94e60f870a356da2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "90b01f27139e40dc927fdf6e5aa342e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ae0c58cbba624797a443905563e080f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9739e2e1aea745b69a83254cd5f2a523": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_aeb693382ec549e4bca658ec73783dda",
              "IPY_MODEL_1a2fb9f460814f8eac60ec125cc0c1a6",
              "IPY_MODEL_43a1f20d4da64b9991ea936c355a2769"
            ],
            "layout": "IPY_MODEL_f4ae9cf0dbcf443e8ed64b41a0fe7e16"
          }
        },
        "aeb693382ec549e4bca658ec73783dda": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_277dcc376f39405cbdb6946c55142dc3",
            "placeholder": "​",
            "style": "IPY_MODEL_95ff5a9715da4c97a6f377ba21b75ee0",
            "value": "Generating train split: "
          }
        },
        "1a2fb9f460814f8eac60ec125cc0c1a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c6cced4fcdbe483da6b2af9a24ce737b",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_53eac8c3218c48fd8822152bf0d23bc6",
            "value": 1
          }
        },
        "43a1f20d4da64b9991ea936c355a2769": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_aacc0f85db1a40b4a1a436ee38f3fa0c",
            "placeholder": "​",
            "style": "IPY_MODEL_ff2e25cf34ec4076b745d4c5676c9ada",
            "value": " 6447/0 [00:00&lt;00:00, 10790.47 examples/s]"
          }
        },
        "f4ae9cf0dbcf443e8ed64b41a0fe7e16": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "277dcc376f39405cbdb6946c55142dc3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "95ff5a9715da4c97a6f377ba21b75ee0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c6cced4fcdbe483da6b2af9a24ce737b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "53eac8c3218c48fd8822152bf0d23bc6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "aacc0f85db1a40b4a1a436ee38f3fa0c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ff2e25cf34ec4076b745d4c5676c9ada": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bc62027581c94c408f1d4a9b68f5f2ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2b61dc787b9d4779871d39d24a5ca4c4",
              "IPY_MODEL_724f638e42dd478b8850973b55671979",
              "IPY_MODEL_9fc79394b83d4db29a19d556c96be232"
            ],
            "layout": "IPY_MODEL_82ccd0e808cc47e181d4e580e666aaf8"
          }
        },
        "2b61dc787b9d4779871d39d24a5ca4c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_38e982c8bdf4489897208cda3c732425",
            "placeholder": "​",
            "style": "IPY_MODEL_4bc6641da2bd4c0896c02f27d0afdee8",
            "value": "Generating train split: "
          }
        },
        "724f638e42dd478b8850973b55671979": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_73f76ac4cfa64e188bb5ca9a3d29862a",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0f27bc4875474e8fa0c7943c96b507e1",
            "value": 1
          }
        },
        "9fc79394b83d4db29a19d556c96be232": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8633ec1c24e8436c92eb6968ae7bec52",
            "placeholder": "​",
            "style": "IPY_MODEL_da3a94d572ac4466a52ba48cbcc8f88c",
            "value": " 6553/0 [00:00&lt;00:00, 11273.18 examples/s]"
          }
        },
        "82ccd0e808cc47e181d4e580e666aaf8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "38e982c8bdf4489897208cda3c732425": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4bc6641da2bd4c0896c02f27d0afdee8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "73f76ac4cfa64e188bb5ca9a3d29862a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "0f27bc4875474e8fa0c7943c96b507e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8633ec1c24e8436c92eb6968ae7bec52": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da3a94d572ac4466a52ba48cbcc8f88c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a09ccf7f81264ca1ab4f502610759c62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b56ba1cee0e245ba9b8241517ff70730",
              "IPY_MODEL_ea5f77c99c5f4004826e57afba3b64cb",
              "IPY_MODEL_30e761b00c6a4a039f3aa12deba101f5"
            ],
            "layout": "IPY_MODEL_747a565cb17f4f1cadce50174b2391db"
          }
        },
        "b56ba1cee0e245ba9b8241517ff70730": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f015a1e9ceba4179b7250013363a8ad0",
            "placeholder": "​",
            "style": "IPY_MODEL_fab4d4526c0c41c4b16074b238085101",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "ea5f77c99c5f4004826e57afba3b64cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_16c4e31234ee4d1dbc2b5a4f4d6c8426",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_dbbc406cddc048a788fed80141bd5ad8",
            "value": 2
          }
        },
        "30e761b00c6a4a039f3aa12deba101f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ca79d08daf174ceab5760242d3e0fe65",
            "placeholder": "​",
            "style": "IPY_MODEL_36d133c170f842d8a27f99f642d593d0",
            "value": " 2/2 [00:10&lt;00:00,  5.11s/it]"
          }
        },
        "747a565cb17f4f1cadce50174b2391db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f015a1e9ceba4179b7250013363a8ad0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fab4d4526c0c41c4b16074b238085101": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "16c4e31234ee4d1dbc2b5a4f4d6c8426": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dbbc406cddc048a788fed80141bd5ad8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ca79d08daf174ceab5760242d3e0fe65": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "36d133c170f842d8a27f99f642d593d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!nvidia-smi"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YtRrFMs9FrLD",
        "outputId": "7a29dc0d-bf6d-45d0-ab40-04c02d70d87f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Tue Apr 30 00:38:44 2024       \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 535.104.05             Driver Version: 535.104.05   CUDA Version: 12.2     |\n",
            "|-----------------------------------------+----------------------+----------------------+\n",
            "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                                         |                      |               MIG M. |\n",
            "|=========================================+======================+======================|\n",
            "|   0  Tesla V100-SXM2-16GB           Off | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   32C    P0              24W / 300W |      0MiB / 16384MiB |      0%      Default |\n",
            "|                                         |                      |                  N/A |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "                                                                                         \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| Processes:                                                                            |\n",
            "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
            "|        ID   ID                                                             Usage      |\n",
            "|=======================================================================================|\n",
            "|  No running processes found                                                           |\n",
            "+---------------------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wvpQBWlf847m",
        "outputId": "ffc7583f-229e-4d48-8253-d6b17cd55c03"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.8/119.8 MB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for transformers (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m297.6/297.6 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for peft (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for accelerate (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m542.0/542.0 kB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m12.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m388.9/388.9 kB\u001b[0m \u001b[31m18.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m245.2/245.2 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m102.0/102.0 kB\u001b[0m \u001b[31m12.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.3/207.3 kB\u001b[0m \u001b[31m19.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m266.8/266.8 kB\u001b[0m \u001b[31m21.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "!pip install -q -U bitsandbytes\n",
        "!pip install -q -U git+https://github.com/huggingface/transformers.git\n",
        "!pip install -q -U git+https://github.com/huggingface/peft.git\n",
        "!pip install -q -U git+https://github.com/huggingface/accelerate.git\n",
        "!pip install -q datasets\n",
        "!pip install -q trl\n",
        "!pip install -q wandb"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "60tSOY9D88nv",
        "outputId": "a1190e6e-ee50-42aa-895b-6b2e3f5e3dbc"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U \"huggingface_hub[cli]\""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xa1_P9v63Sf4",
        "outputId": "6b59586a-a090-4c10-8f2e-6e4369428e65"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: huggingface_hub[cli] in /usr/local/lib/python3.10/dist-packages (0.22.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub[cli]) (3.13.4)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub[cli]) (2023.6.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub[cli]) (24.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub[cli]) (6.0.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface_hub[cli]) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub[cli]) (4.66.2)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub[cli]) (4.11.0)\n",
            "Requirement already satisfied: InquirerPy==0.3.4 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub[cli]) (0.3.4)\n",
            "Requirement already satisfied: pfzy<0.4.0,>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from InquirerPy==0.3.4->huggingface_hub[cli]) (0.3.4)\n",
            "Requirement already satisfied: prompt-toolkit<4.0.0,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from InquirerPy==0.3.4->huggingface_hub[cli]) (3.0.43)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub[cli]) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub[cli]) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub[cli]) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub[cli]) (2024.2.2)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt-toolkit<4.0.0,>=3.0.1->InquirerPy==0.3.4->huggingface_hub[cli]) (0.2.13)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 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\n",
        "!huggingface-cli login"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mNrYKr7_3UVT",
        "outputId": "a93927f3-83c6-423c-c50c-3199916a3ab5"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "    _|    _|  _|    _|    _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|_|_|_|    _|_|      _|_|_|  _|_|_|_|\n",
            "    _|    _|  _|    _|  _|        _|          _|    _|_|    _|  _|            _|        _|    _|  _|        _|\n",
            "    _|_|_|_|  _|    _|  _|  _|_|  _|  _|_|    _|    _|  _|  _|  _|  _|_|      _|_|_|    _|_|_|_|  _|        _|_|_|\n",
            "    _|    _|  _|    _|  _|    _|  _|    _|    _|    _|    _|_|  _|    _|      _|        _|    _|  _|        _|\n",
            "    _|    _|    _|_|      _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|        _|    _|    _|_|_|  _|_|_|_|\n",
            "\n",
            "    A token is already saved on your machine. Run `huggingface-cli whoami` to get more information or `huggingface-cli logout` if you want to log out.\n",
            "    Setting a new token will erase the existing one.\n",
            "    To login, `huggingface_hub` requires a token generated from https://huggingface.co/settings/tokens .\n",
            "Enter your token (input will not be visible): \n",
            "Add token as git credential? (Y/n) y\n",
            "Token is valid (permission: write).\n",
            "\u001b[1m\u001b[31mCannot authenticate through git-credential as no helper is defined on your machine.\n",
            "You might have to re-authenticate when pushing to the Hugging Face Hub.\n",
            "Run the following command in your terminal in case you want to set the 'store' credential helper as default.\n",
            "\n",
            "git config --global credential.helper store\n",
            "\n",
            "Read https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage for more details.\u001b[0m\n",
            "Token has not been saved to git credential helper.\n",
            "Your token has been saved to /root/.cache/huggingface/token\n",
            "Login successful\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# !huggingface-cli download mistralai/Mistral-7B-v0.1 #--local-dir /content/drive/MyDrive/ConceptualGapsProject/huggingFace/ #--cache-dir /content/drive/MyDrive/ConceptualGapsProject/huggingFace/cache/"
      ],
      "metadata": {
        "id": "mMZU2rZS3dgB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import wandb\n",
        "import random\n",
        "from datetime import datetime"
      ],
      "metadata": {
        "id": "h_leDRZHBPCY"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# c6ca2911139daf1b55ae8d277c3b9bdacc4d8608\n",
        "wandb.login()"
      ],
      "metadata": {
        "id": "QV8MUXkmFQel",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0396b070-a188-4552-f7d6-a65f85f1492d"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33max2119\u001b[0m (\u001b[33mmyprojs\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "run = wandb.init(\n",
        "    # Set the project where this run will be logged\n",
        "    project=\"MistralDPO_Reddit_3\",\n",
        ")"
      ],
      "metadata": {
        "id": "qgUIuNxdFRXu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "outputId": "adc734a0-2c4a-4ebf-cc60-5e0474c9eb40"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Tracking run with wandb version 0.16.6"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Run data is saved locally in <code>/content/wandb/run-20240430_081813-4b4u7vja</code>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Syncing run <strong><a href='https://wandb.ai/myprojs/MistralDPO_Reddit_3/runs/4b4u7vja' target=\"_blank\">swift-rain-28</a></strong> to <a href='https://wandb.ai/myprojs/MistralDPO_Reddit_3' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              " View project at <a href='https://wandb.ai/myprojs/MistralDPO_Reddit_3' target=\"_blank\">https://wandb.ai/myprojs/MistralDPO_Reddit_3</a>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              " View run at <a href='https://wandb.ai/myprojs/MistralDPO_Reddit_3/runs/4b4u7vja' target=\"_blank\">https://wandb.ai/myprojs/MistralDPO_Reddit_3/runs/4b4u7vja</a>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# JSON Formatting"
      ],
      "metadata": {
        "id": "MmWV2PHSfUfw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def formatting_prompts_func_3(example):\n",
        "    bos_token = \"<s>\"\n",
        "    eos_token = \"</s>\"\n",
        "    text = \"\"\n",
        "    text += bos_token\n",
        "    text += f\"SUBREDDIT: r/{example['subreddit']}\\nTITLE: {example['title']}\\nPOST: {example['post']}\\nTL;DR: \"\n",
        "    # text += eos_token\n",
        "    return text"
      ],
      "metadata": {
        "id": "p85WtKzIV4fI"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# import json\n",
        "# import os\n",
        "# from tqdm import tqdm\n",
        "\n",
        "# # Path to the JSON file\n",
        "# directory_path = '/content/drive/MyDrive/ConceptualGapsProject/redditDataset/DPOdataset/comparisons/'\n",
        "\n",
        "# # Dictionaries to hold segregated data\n",
        "# json_data_train = []\n",
        "# json_data_valid1 = []\n",
        "# json_data_valid2 = []\n",
        "\n",
        "# json_train_len = 0\n",
        "# json_valid1_len = 0\n",
        "# json_valid2_len = 0\n",
        "# len_overall = 0\n",
        "\n",
        "# json_ids_dict = []\n",
        "\n",
        "# # List of file names (you will have to fill this with the actual file names)\n",
        "# file_names = [f\"batch{i}.json\" for i in range(3,23) if i != 21]  # Add all file names here\n",
        "\n",
        "# # Iterate over each file name\n",
        "# for file_name in tqdm(file_names):\n",
        "#     file_path = os.path.join(directory_path, file_name)\n",
        "\n",
        "#     # Open and read the JSON file line by line\n",
        "#     with open(file_path, 'r') as file:\n",
        "#         for line in file:\n",
        "#             len_overall+=1\n",
        "#             try:\n",
        "#                 # Load each line as a separate JSON object\n",
        "#                 json_data = json.loads(line)\n",
        "#                 # if json_data['info']['id'] not in json_ids_dict:\n",
        "#                 #   json_ids_dict.append(json_data['info']['id'])\n",
        "#                 # else:\n",
        "#                 #   # print(\"Came here\")\n",
        "#                 #   continue\n",
        "#                 # Segregate data based on the 'split' key\n",
        "#                 if json_data['split'] == 'train':\n",
        "#                     pref_json_data = {\"prompt\" : formatting_prompts_func_3(json_data['info']), \"chosen\" : json_data['summaries'][json_data['choice']]['text'], \"rejected\" : json_data['summaries'][abs(json_data['choice']-1)]['text']}\n",
        "#                     json_data_train.append(pref_json_data)\n",
        "#                     json_train_len+=1\n",
        "#                 elif json_data['split'] == 'valid1':\n",
        "#                     pref_json_data = {\"prompt\" : formatting_prompts_func_3(json_data['info']), \"chosen\" : json_data['summaries'][json_data['choice']]['text'], \"rejected\" : json_data['summaries'][abs(json_data['choice']-1)]['text']}\n",
        "#                     json_data_valid1.append(pref_json_data)\n",
        "#                     # json_data['split'] = 'valid'\n",
        "#                     json_valid1_len+=1\n",
        "#                 elif json_data['split'] == 'valid2':\n",
        "#                     pref_json_data = {\"prompt\" : formatting_prompts_func_3(json_data['info']), \"chosen\" : json_data['summaries'][json_data['choice']]['text'], \"rejected\" : json_data['summaries'][abs(json_data['choice']-1)]['text']}\n",
        "#                     json_data_valid2.append(pref_json_data)\n",
        "#                     # json_data['split'] = 'test'\n",
        "#                     json_valid2_len+=1\n",
        "#             except json.JSONDecodeError as e:\n",
        "#                 print(f\"Error decoding JSON in {file_name}: {e}\")\n",
        "\n",
        "# # Function to save segregated data to separate JSON files\n",
        "# def save_json(data, filename):\n",
        "#     with open(os.path.join(directory_path, filename), 'w') as file:\n",
        "#         json.dump(data, file, indent=4)\n",
        "\n",
        "# # Save the segregated data to separate JSON files\n",
        "# save_json(json_data_train, directory_path + 'reddit_batch_all_train.json')\n",
        "# save_json(json_data_valid1, directory_path + 'reddit_batch_all_valid1.json')\n",
        "# save_json(json_data_valid2, directory_path + 'reddit_batch_all_valid2.json')\n",
        "\n",
        "# print(\"Data from all files segregated and saved into separate files.\")\n",
        "# print(\"len_overall : \",len_overall)\n",
        "# print(\"json_train_len : \",json_train_len)\n",
        "# print(\"json_valid1_len : \",json_valid1_len)\n",
        "# print(\"json_valid2_len : \",json_valid2_len)"
      ],
      "metadata": {
        "id": "EqDIQypESGOK"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Loading Dataset"
      ],
      "metadata": {
        "id": "uaXJkXQafXv3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Dataset\n",
        "from datasets import load_dataset\n",
        "from trl import SFTTrainer\n",
        "\n",
        "print(\"Loading Dataset\\n\")\n",
        "\n",
        "# load train jsonl dataset\n",
        "train_dataset = load_dataset(\"json\", data_files=\"/content/drive/MyDrive/ConceptualGapsProject/redditDataset/DPOdataset/comparisons/reddit_batch_all_train.json\", split=\"train\")\n",
        "print(\"train_dataset : \",train_dataset)\n",
        "# load valid jsonl dataset\n",
        "valid_dataset = load_dataset(\"json\", data_files=\"/content/drive/MyDrive/ConceptualGapsProject/redditDataset/DPOdataset/comparisons/reddit_batch_all_valid1.json\", split=\"train\")\n",
        "print(\"valid_dataset : \",valid_dataset)\n",
        "# load test jsonl dataset\n",
        "test_dataset = load_dataset(\"json\", data_files=\"/content/drive/MyDrive/ConceptualGapsProject/redditDataset/DPOdataset/comparisons/reddit_batch_all_valid2.json\", split=\"train[:10%]\")\n",
        "print(\"test_dataset : \",test_dataset)"
      ],
      "metadata": {
        "id": "3HlVdMu39EjX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d4f84c3b-c253-4a2d-87fb-c69464be15a0"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading Dataset\n",
            "\n",
            "train_dataset :  Dataset({\n",
            "    features: ['rejected', 'prompt', 'chosen'],\n",
            "    num_rows: 92858\n",
            "})\n",
            "valid_dataset :  Dataset({\n",
            "    features: ['rejected', 'prompt', 'chosen'],\n",
            "    num_rows: 33082\n",
            "})\n",
            "test_dataset :  Dataset({\n",
            "    features: ['rejected', 'prompt', 'chosen'],\n",
            "    num_rows: 5072\n",
            "})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WSabgew31yx_",
        "outputId": "87a4781e-01b3-4c58-ba76-b8f7986e6ef3"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['rejected', 'prompt', 'chosen'],\n",
              "    num_rows: 92858\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model and Tokenizer"
      ],
      "metadata": {
        "id": "eR1cIWMtfaUJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Model\n",
        "\n",
        "print(\"Loading Model\\n\")\n",
        "\n",
        "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
        "import torch\n",
        "\n",
        "nf4_config = BitsAndBytesConfig(\n",
        "   load_in_4bit=True,\n",
        "   llm_int8_threshold=6.0,\n",
        "   llm_int8_has_fp16_weight=False,\n",
        "   bnb_4bit_quant_type=\"nf4\",\n",
        "   bnb_4bit_use_double_quant=True,\n",
        "   bnb_4bit_compute_dtype=torch.float16\n",
        ")\n",
        "\n",
        "model = AutoModelForCausalLM.from_pretrained(\n",
        "    # \"/content/drive/MyDrive/ConceptualGapsProject/huggingFace\",\n",
        "    \"mistralai/Mistral-7B-v0.1\",\n",
        "    cache_dir = \"/content/mistral_weights/\",\n",
        "    # load_in_4bit=True,\n",
        "    device_map='auto',\n",
        "    quantization_config=nf4_config,\n",
        "    # use_cache=True,\n",
        "    torch_dtype=torch.float16,\n",
        ")\n",
        "model.config.use_cache = False"
      ],
      "metadata": {
        "id": "P_bpCZkN9PoO",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 188,
          "referenced_widgets": [
            "5b688adca60f44588e4a3d564422911d",
            "28bab1e70714420cb57a5fb8109d9031",
            "574454f2f9974f04a0c7ee0f697e5df4",
            "b7ad4e45448849f99b4ab121a58e16b0",
            "0b210649cab24fcd84668179a8ba17b2",
            "df165e8954494419a09ad57cdc8afa0b",
            "d0c67c0da65c42e380bd5670aa99be0b",
            "ecb6c568610040368f4be9156851c8a5",
            "50d2c3a5e5764c828f03a08e4465c6dd",
            "485a2162bacb4047b2362ff5757414be",
            "876ac1c941cb44028e55246f6423cc20"
          ]
        },
        "outputId": "bdeedb28-0cd5-4b4f-9cfa-7af9ec5803d0"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading Model\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5b688adca60f44588e4a3d564422911d"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Tokenizer\n",
        "print(\"Loading Tokenizer\\n\")\n",
        "\n",
        "from transformers import AutoTokenizer\n",
        "\n",
        "# tokenizer = AutoTokenizer.from_pretrained(\"/content/drive/MyDrive/ConceptualGapsProject/huggingFace\", trust_remote_code=True,)\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"mistralai/Mistral-7B-v0.1\")\n",
        "\n",
        "tokenizer.pad_token = tokenizer.eos_token\n",
        "tokenizer.padding_side = \"left\""
      ],
      "metadata": {
        "id": "U8uGRiwZ9S4z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b0985f14-2870-4249-9b8e-8ae051320be5"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading Tokenizer\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Loading PEFT model"
      ],
      "metadata": {
        "id": "r0zVtYg_fdxg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def print_trainable_parameters(model):\n",
        "    \"\"\"\n",
        "    Prints the number of trainable parameters in the model.\n",
        "    \"\"\"\n",
        "    trainable_params = 0\n",
        "    all_param = 0\n",
        "    for _, param in model.named_parameters():\n",
        "        all_param += param.numel()\n",
        "        if param.requires_grad:\n",
        "            trainable_params += param.numel()\n",
        "    print(\n",
        "        f\"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}\"\n",
        "    )"
      ],
      "metadata": {
        "id": "lgF7R8-9EIsz"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# if hasattr(model, \"enable_input_require_grads\"):\n",
        "#     print(\"enable_input_require_grads\")\n",
        "#     model.enable_input_require_grads()\n",
        "# else:\n",
        "#     def make_inputs_require_grad(module, input, output):\n",
        "#          output.requires_grad_(True)\n",
        "\n",
        "#     model.get_input_embeddings().register_forward_hook(make_inputs_require_grad)"
      ],
      "metadata": {
        "id": "5KiJh7_s-peO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# from peft import AutoPeftModelForCausalLM, LoraConfig, get_peft_model, prepare_model_for_kbit_training\n",
        "\n",
        "# peft_config = LoraConfig(\n",
        "#     lora_alpha=16,\n",
        "#     lora_dropout=0.1,\n",
        "#     r=64,\n",
        "#     # target_modules=[\n",
        "#     #     \"q_proj\",\n",
        "#     #     \"k_proj\",\n",
        "#     #     \"v_proj\",\n",
        "#     #     \"o_proj\",\n",
        "#     #     \"gate_proj\",\n",
        "#     #     \"up_proj\",\n",
        "#     #     \"down_proj\",\n",
        "#     #     \"lm_head\",\n",
        "#     # ],\n",
        "#     bias=\"none\",\n",
        "#     task_type=\"CAUSAL_LM\"\n",
        "# )"
      ],
      "metadata": {
        "id": "dotxjklAOPsT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Loading PEFT Model\n",
        "from peft import PeftModel\n",
        "\n",
        "model = PeftModel.from_pretrained(model, \"/content/drive/MyDrive/ConceptualGapsProject/SFT_Model_3/checkpoint-10000/\", is_trainable = True, adapter_name=\"train_1\")\n",
        "\n",
        "print_trainable_parameters(model)"
      ],
      "metadata": {
        "id": "JG9KQOk59V8I",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f21b871f-d473-4a31-db1f-2e6de86119fc"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "trainable params: 27262976 || all params: 3779334144 || trainable%: 0.7213698223345028\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.get_nb_trainable_parameters()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_UwKo0em-ANM",
        "outputId": "5d169b37-7541-4d8f-ebef-5e83b27303b9"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(27262976, 7268995072)"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# sft_ref_model_3 = PeftModel.from_pretrained(model, \"/content/drive/MyDrive/ConceptualGapsProject/SFT_Model_3/checkpoint-10000/\", is_trainable = True)\n",
        "# # sft_ref_model_3 = prepare_model_for_kbit_training(sft_ref_model_3)\n",
        "# # sft_ref_model_3 = get_peft_model(sft_ref_model_3, peft_config)\n",
        "\n",
        "# print_trainable_parameters(sft_ref_model_3)\n",
        "\n",
        "# Load the adapter a second time, with a different name, which will be our reference model.\n",
        "model.load_adapter(\"/content/drive/MyDrive/ConceptualGapsProject/SFT_Model_3/checkpoint-10000/\", adapter_name=\"reference\")"
      ],
      "metadata": {
        "id": "h1w24Vi_EJre",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "46c5c3f4-547f-46e4-df9a-aa9f7937cef6"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "_IncompatibleKeys(missing_keys=['base_model.model.model.embed_tokens.weight', 'base_model.model.model.layers.0.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.0.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.0.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.0.self_attn.k_proj.weight', 'base_model.model.model.layers.0.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.0.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.0.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.0.self_attn.o_proj.weight', 'base_model.model.model.layers.0.mlp.gate_proj.weight', 'base_model.model.model.layers.0.mlp.up_proj.weight', 'base_model.model.model.layers.0.mlp.down_proj.weight', 'base_model.model.model.layers.0.input_layernorm.weight', 'base_model.model.model.layers.0.post_attention_layernorm.weight', 'base_model.model.model.layers.1.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.1.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.1.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.1.self_attn.k_proj.weight', 'base_model.model.model.layers.1.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.1.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.1.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.1.self_attn.o_proj.weight', 'base_model.model.model.layers.1.mlp.gate_proj.weight', 'base_model.model.model.layers.1.mlp.up_proj.weight', 'base_model.model.model.layers.1.mlp.down_proj.weight', 'base_model.model.model.layers.1.input_layernorm.weight', 'base_model.model.model.layers.1.post_attention_layernorm.weight', 'base_model.model.model.layers.2.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.2.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.2.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.2.self_attn.k_proj.weight', 'base_model.model.model.layers.2.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.2.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.2.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.2.self_attn.o_proj.weight', 'base_model.model.model.layers.2.mlp.gate_proj.weight', 'base_model.model.model.layers.2.mlp.up_proj.weight', 'base_model.model.model.layers.2.mlp.down_proj.weight', 'base_model.model.model.layers.2.input_layernorm.weight', 'base_model.model.model.layers.2.post_attention_layernorm.weight', 'base_model.model.model.layers.3.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.3.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.3.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.3.self_attn.k_proj.weight', 'base_model.model.model.layers.3.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.3.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.3.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.3.self_attn.o_proj.weight', 'base_model.model.model.layers.3.mlp.gate_proj.weight', 'base_model.model.model.layers.3.mlp.up_proj.weight', 'base_model.model.model.layers.3.mlp.down_proj.weight', 'base_model.model.model.layers.3.input_layernorm.weight', 'base_model.model.model.layers.3.post_attention_layernorm.weight', 'base_model.model.model.layers.4.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.4.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.4.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.4.self_attn.k_proj.weight', 'base_model.model.model.layers.4.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.4.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.4.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.4.self_attn.o_proj.weight', 'base_model.model.model.layers.4.mlp.gate_proj.weight', 'base_model.model.model.layers.4.mlp.up_proj.weight', 'base_model.model.model.layers.4.mlp.down_proj.weight', 'base_model.model.model.layers.4.input_layernorm.weight', 'base_model.model.model.layers.4.post_attention_layernorm.weight', 'base_model.model.model.layers.5.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.5.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.5.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.5.self_attn.k_proj.weight', 'base_model.model.model.layers.5.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.5.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.5.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.5.self_attn.o_proj.weight', 'base_model.model.model.layers.5.mlp.gate_proj.weight', 'base_model.model.model.layers.5.mlp.up_proj.weight', 'base_model.model.model.layers.5.mlp.down_proj.weight', 'base_model.model.model.layers.5.input_layernorm.weight', 'base_model.model.model.layers.5.post_attention_layernorm.weight', 'base_model.model.model.layers.6.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.6.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.6.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.6.self_attn.k_proj.weight', 'base_model.model.model.layers.6.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.6.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.6.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.6.self_attn.o_proj.weight', 'base_model.model.model.layers.6.mlp.gate_proj.weight', 'base_model.model.model.layers.6.mlp.up_proj.weight', 'base_model.model.model.layers.6.mlp.down_proj.weight', 'base_model.model.model.layers.6.input_layernorm.weight', 'base_model.model.model.layers.6.post_attention_layernorm.weight', 'base_model.model.model.layers.7.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.7.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.7.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.7.self_attn.k_proj.weight', 'base_model.model.model.layers.7.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.7.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.7.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.7.self_attn.o_proj.weight', 'base_model.model.model.layers.7.mlp.gate_proj.weight', 'base_model.model.model.layers.7.mlp.up_proj.weight', 'base_model.model.model.layers.7.mlp.down_proj.weight', 'base_model.model.model.layers.7.input_layernorm.weight', 'base_model.model.model.layers.7.post_attention_layernorm.weight', 'base_model.model.model.layers.8.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.8.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.8.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.8.self_attn.k_proj.weight', 'base_model.model.model.layers.8.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.8.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.8.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.8.self_attn.o_proj.weight', 'base_model.model.model.layers.8.mlp.gate_proj.weight', 'base_model.model.model.layers.8.mlp.up_proj.weight', 'base_model.model.model.layers.8.mlp.down_proj.weight', 'base_model.model.model.layers.8.input_layernorm.weight', 'base_model.model.model.layers.8.post_attention_layernorm.weight', 'base_model.model.model.layers.9.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.9.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.9.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.9.self_attn.k_proj.weight', 'base_model.model.model.layers.9.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.9.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.9.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.9.self_attn.o_proj.weight', 'base_model.model.model.layers.9.mlp.gate_proj.weight', 'base_model.model.model.layers.9.mlp.up_proj.weight', 'base_model.model.model.layers.9.mlp.down_proj.weight', 'base_model.model.model.layers.9.input_layernorm.weight', 'base_model.model.model.layers.9.post_attention_layernorm.weight', 'base_model.model.model.layers.10.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.10.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.10.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.10.self_attn.k_proj.weight', 'base_model.model.model.layers.10.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.10.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.10.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.10.self_attn.o_proj.weight', 'base_model.model.model.layers.10.mlp.gate_proj.weight', 'base_model.model.model.layers.10.mlp.up_proj.weight', 'base_model.model.model.layers.10.mlp.down_proj.weight', 'base_model.model.model.layers.10.input_layernorm.weight', 'base_model.model.model.layers.10.post_attention_layernorm.weight', 'base_model.model.model.layers.11.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.11.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.11.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.11.self_attn.k_proj.weight', 'base_model.model.model.layers.11.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.11.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.11.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.11.self_attn.o_proj.weight', 'base_model.model.model.layers.11.mlp.gate_proj.weight', 'base_model.model.model.layers.11.mlp.up_proj.weight', 'base_model.model.model.layers.11.mlp.down_proj.weight', 'base_model.model.model.layers.11.input_layernorm.weight', 'base_model.model.model.layers.11.post_attention_layernorm.weight', 'base_model.model.model.layers.12.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.12.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.12.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.12.self_attn.k_proj.weight', 'base_model.model.model.layers.12.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.12.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.12.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.12.self_attn.o_proj.weight', 'base_model.model.model.layers.12.mlp.gate_proj.weight', 'base_model.model.model.layers.12.mlp.up_proj.weight', 'base_model.model.model.layers.12.mlp.down_proj.weight', 'base_model.model.model.layers.12.input_layernorm.weight', 'base_model.model.model.layers.12.post_attention_layernorm.weight', 'base_model.model.model.layers.13.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.13.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.13.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.13.self_attn.k_proj.weight', 'base_model.model.model.layers.13.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.13.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.13.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.13.self_attn.o_proj.weight', 'base_model.model.model.layers.13.mlp.gate_proj.weight', 'base_model.model.model.layers.13.mlp.up_proj.weight', 'base_model.model.model.layers.13.mlp.down_proj.weight', 'base_model.model.model.layers.13.input_layernorm.weight', 'base_model.model.model.layers.13.post_attention_layernorm.weight', 'base_model.model.model.layers.14.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.14.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.14.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.14.self_attn.k_proj.weight', 'base_model.model.model.layers.14.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.14.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.14.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.14.self_attn.o_proj.weight', 'base_model.model.model.layers.14.mlp.gate_proj.weight', 'base_model.model.model.layers.14.mlp.up_proj.weight', 'base_model.model.model.layers.14.mlp.down_proj.weight', 'base_model.model.model.layers.14.input_layernorm.weight', 'base_model.model.model.layers.14.post_attention_layernorm.weight', 'base_model.model.model.layers.15.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.15.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.15.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.15.self_attn.k_proj.weight', 'base_model.model.model.layers.15.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.15.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.15.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.15.self_attn.o_proj.weight', 'base_model.model.model.layers.15.mlp.gate_proj.weight', 'base_model.model.model.layers.15.mlp.up_proj.weight', 'base_model.model.model.layers.15.mlp.down_proj.weight', 'base_model.model.model.layers.15.input_layernorm.weight', 'base_model.model.model.layers.15.post_attention_layernorm.weight', 'base_model.model.model.layers.16.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.16.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.16.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.16.self_attn.k_proj.weight', 'base_model.model.model.layers.16.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.16.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.16.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.16.self_attn.o_proj.weight', 'base_model.model.model.layers.16.mlp.gate_proj.weight', 'base_model.model.model.layers.16.mlp.up_proj.weight', 'base_model.model.model.layers.16.mlp.down_proj.weight', 'base_model.model.model.layers.16.input_layernorm.weight', 'base_model.model.model.layers.16.post_attention_layernorm.weight', 'base_model.model.model.layers.17.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.17.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.17.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.17.self_attn.k_proj.weight', 'base_model.model.model.layers.17.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.17.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.17.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.17.self_attn.o_proj.weight', 'base_model.model.model.layers.17.mlp.gate_proj.weight', 'base_model.model.model.layers.17.mlp.up_proj.weight', 'base_model.model.model.layers.17.mlp.down_proj.weight', 'base_model.model.model.layers.17.input_layernorm.weight', 'base_model.model.model.layers.17.post_attention_layernorm.weight', 'base_model.model.model.layers.18.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.18.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.18.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.18.self_attn.k_proj.weight', 'base_model.model.model.layers.18.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.18.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.18.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.18.self_attn.o_proj.weight', 'base_model.model.model.layers.18.mlp.gate_proj.weight', 'base_model.model.model.layers.18.mlp.up_proj.weight', 'base_model.model.model.layers.18.mlp.down_proj.weight', 'base_model.model.model.layers.18.input_layernorm.weight', 'base_model.model.model.layers.18.post_attention_layernorm.weight', 'base_model.model.model.layers.19.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.19.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.19.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.19.self_attn.k_proj.weight', 'base_model.model.model.layers.19.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.19.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.19.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.19.self_attn.o_proj.weight', 'base_model.model.model.layers.19.mlp.gate_proj.weight', 'base_model.model.model.layers.19.mlp.up_proj.weight', 'base_model.model.model.layers.19.mlp.down_proj.weight', 'base_model.model.model.layers.19.input_layernorm.weight', 'base_model.model.model.layers.19.post_attention_layernorm.weight', 'base_model.model.model.layers.20.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.20.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.20.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.20.self_attn.k_proj.weight', 'base_model.model.model.layers.20.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.20.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.20.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.20.self_attn.o_proj.weight', 'base_model.model.model.layers.20.mlp.gate_proj.weight', 'base_model.model.model.layers.20.mlp.up_proj.weight', 'base_model.model.model.layers.20.mlp.down_proj.weight', 'base_model.model.model.layers.20.input_layernorm.weight', 'base_model.model.model.layers.20.post_attention_layernorm.weight', 'base_model.model.model.layers.21.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.21.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.21.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.21.self_attn.k_proj.weight', 'base_model.model.model.layers.21.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.21.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.21.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.21.self_attn.o_proj.weight', 'base_model.model.model.layers.21.mlp.gate_proj.weight', 'base_model.model.model.layers.21.mlp.up_proj.weight', 'base_model.model.model.layers.21.mlp.down_proj.weight', 'base_model.model.model.layers.21.input_layernorm.weight', 'base_model.model.model.layers.21.post_attention_layernorm.weight', 'base_model.model.model.layers.22.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.22.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.22.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.22.self_attn.k_proj.weight', 'base_model.model.model.layers.22.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.22.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.22.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.22.self_attn.o_proj.weight', 'base_model.model.model.layers.22.mlp.gate_proj.weight', 'base_model.model.model.layers.22.mlp.up_proj.weight', 'base_model.model.model.layers.22.mlp.down_proj.weight', 'base_model.model.model.layers.22.input_layernorm.weight', 'base_model.model.model.layers.22.post_attention_layernorm.weight', 'base_model.model.model.layers.23.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.23.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.23.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.23.self_attn.k_proj.weight', 'base_model.model.model.layers.23.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.23.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.23.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.23.self_attn.o_proj.weight', 'base_model.model.model.layers.23.mlp.gate_proj.weight', 'base_model.model.model.layers.23.mlp.up_proj.weight', 'base_model.model.model.layers.23.mlp.down_proj.weight', 'base_model.model.model.layers.23.input_layernorm.weight', 'base_model.model.model.layers.23.post_attention_layernorm.weight', 'base_model.model.model.layers.24.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.24.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.24.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.24.self_attn.k_proj.weight', 'base_model.model.model.layers.24.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.24.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.24.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.24.self_attn.o_proj.weight', 'base_model.model.model.layers.24.mlp.gate_proj.weight', 'base_model.model.model.layers.24.mlp.up_proj.weight', 'base_model.model.model.layers.24.mlp.down_proj.weight', 'base_model.model.model.layers.24.input_layernorm.weight', 'base_model.model.model.layers.24.post_attention_layernorm.weight', 'base_model.model.model.layers.25.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.25.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.25.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.25.self_attn.k_proj.weight', 'base_model.model.model.layers.25.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.25.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.25.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.25.self_attn.o_proj.weight', 'base_model.model.model.layers.25.mlp.gate_proj.weight', 'base_model.model.model.layers.25.mlp.up_proj.weight', 'base_model.model.model.layers.25.mlp.down_proj.weight', 'base_model.model.model.layers.25.input_layernorm.weight', 'base_model.model.model.layers.25.post_attention_layernorm.weight', 'base_model.model.model.layers.26.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.26.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.26.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.26.self_attn.k_proj.weight', 'base_model.model.model.layers.26.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.26.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.26.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.26.self_attn.o_proj.weight', 'base_model.model.model.layers.26.mlp.gate_proj.weight', 'base_model.model.model.layers.26.mlp.up_proj.weight', 'base_model.model.model.layers.26.mlp.down_proj.weight', 'base_model.model.model.layers.26.input_layernorm.weight', 'base_model.model.model.layers.26.post_attention_layernorm.weight', 'base_model.model.model.layers.27.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.27.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.27.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.27.self_attn.k_proj.weight', 'base_model.model.model.layers.27.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.27.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.27.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.27.self_attn.o_proj.weight', 'base_model.model.model.layers.27.mlp.gate_proj.weight', 'base_model.model.model.layers.27.mlp.up_proj.weight', 'base_model.model.model.layers.27.mlp.down_proj.weight', 'base_model.model.model.layers.27.input_layernorm.weight', 'base_model.model.model.layers.27.post_attention_layernorm.weight', 'base_model.model.model.layers.28.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.28.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.28.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.28.self_attn.k_proj.weight', 'base_model.model.model.layers.28.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.28.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.28.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.28.self_attn.o_proj.weight', 'base_model.model.model.layers.28.mlp.gate_proj.weight', 'base_model.model.model.layers.28.mlp.up_proj.weight', 'base_model.model.model.layers.28.mlp.down_proj.weight', 'base_model.model.model.layers.28.input_layernorm.weight', 'base_model.model.model.layers.28.post_attention_layernorm.weight', 'base_model.model.model.layers.29.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.29.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.29.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.29.self_attn.k_proj.weight', 'base_model.model.model.layers.29.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.29.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.29.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.29.self_attn.o_proj.weight', 'base_model.model.model.layers.29.mlp.gate_proj.weight', 'base_model.model.model.layers.29.mlp.up_proj.weight', 'base_model.model.model.layers.29.mlp.down_proj.weight', 'base_model.model.model.layers.29.input_layernorm.weight', 'base_model.model.model.layers.29.post_attention_layernorm.weight', 'base_model.model.model.layers.30.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.30.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.30.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.30.self_attn.k_proj.weight', 'base_model.model.model.layers.30.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.30.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.30.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.30.self_attn.o_proj.weight', 'base_model.model.model.layers.30.mlp.gate_proj.weight', 'base_model.model.model.layers.30.mlp.up_proj.weight', 'base_model.model.model.layers.30.mlp.down_proj.weight', 'base_model.model.model.layers.30.input_layernorm.weight', 'base_model.model.model.layers.30.post_attention_layernorm.weight', 'base_model.model.model.layers.31.self_attn.q_proj.base_layer.weight', 'base_model.model.model.layers.31.self_attn.q_proj.lora_A.train_1.weight', 'base_model.model.model.layers.31.self_attn.q_proj.lora_B.train_1.weight', 'base_model.model.model.layers.31.self_attn.k_proj.weight', 'base_model.model.model.layers.31.self_attn.v_proj.base_layer.weight', 'base_model.model.model.layers.31.self_attn.v_proj.lora_A.train_1.weight', 'base_model.model.model.layers.31.self_attn.v_proj.lora_B.train_1.weight', 'base_model.model.model.layers.31.self_attn.o_proj.weight', 'base_model.model.model.layers.31.mlp.gate_proj.weight', 'base_model.model.model.layers.31.mlp.up_proj.weight', 'base_model.model.model.layers.31.mlp.down_proj.weight', 'base_model.model.model.layers.31.input_layernorm.weight', 'base_model.model.model.layers.31.post_attention_layernorm.weight', 'base_model.model.model.norm.weight', 'base_model.model.lm_head.weight'], unexpected_keys=[])"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# DPO Training"
      ],
      "metadata": {
        "id": "hP5TVk2MfgAN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# DPO Training Arguments\n",
        "from transformers import TrainingArguments\n",
        "\n",
        "args = TrainingArguments(\n",
        "  output_dir = \"/content/drive/MyDrive/ConceptualGapsProject/DPOcheckpoints/MistralDPORedditSumm_3_2\",\n",
        "  save_steps=500,\n",
        "  # num_train_epochs=7,\n",
        "  max_steps = 40000, # comment out this line if you want to train in epochs\n",
        "  gradient_accumulation_steps=1,\n",
        "  gradient_checkpointing=True,\n",
        "  per_device_train_batch_size = 4,\n",
        "  warmup_steps = 50,\n",
        "  logging_steps=10,\n",
        "  save_strategy=\"steps\",\n",
        "  save_total_limit=10,\n",
        "  optim=\"paged_adamw_32bit\",\n",
        "  # load_best_model_at_end= True,\n",
        "  report_to=\"wandb\",\n",
        "  do_eval=True,\n",
        "  run_name = f\"{datetime.now().strftime('%Y-%m-%d-%H-%M')}\",\n",
        "  #evaluation_strategy=\"epoch\",\n",
        "  evaluation_strategy=\"steps\",\n",
        "  eval_steps=500, # comment out this line if you want to evaluate at the end of each epoch\n",
        "  learning_rate=2e-4,\n",
        "  bf16=False,\n",
        "  fp16=True,\n",
        "  # lr_scheduler_type='constant',\n",
        ")"
      ],
      "metadata": {
        "id": "56Wd2ztn9bXl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6cc604af-137c-4bce-f019-dbbe9e2becd4"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1463: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataset = train_dataset.shard(num_shards=10, index=3)\n",
        "eval_dataset = test_dataset.shard(num_shards=10, index=3)"
      ],
      "metadata": {
        "id": "pKv3nk7Hr8kC"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# DPO Training\n",
        "\n",
        "from trl import DPOTrainer\n",
        "\n",
        "dpo_trainer=DPOTrainer(\n",
        "    model,\n",
        "    ref_model=None,\n",
        "    args=args,\n",
        "    train_dataset=train_dataset,\n",
        "    eval_dataset = eval_dataset,\n",
        "    tokenizer=tokenizer,\n",
        "    # peft_config=peft_config,\n",
        "    beta=0.1,\n",
        "    # max_prompt_length=1024,\n",
        "    # max_length=1536,\n",
        "    model_adapter_name=\"train_1\",\n",
        "    ref_adapter_name=\"reference\",\n",
        ")\n",
        "\n",
        "dpo_trainer.train()"
      ],
      "metadata": {
        "id": "PBP6AG1gBXwN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "870e76e22db24d87a915ec1892e13d9e",
            "0f0b8406b6064d4e8214c8c6bfe3c8f7",
            "7ec4e356994e49718507b3ad05f9ecef",
            "e90285eeeb4547ada23ea7b77d017485",
            "3437b1214fb1460c92c858acf5eb0106",
            "57a6625d1dda4bd19c738fddd1890fd2",
            "c3b58d9680ab4d82874ab61b1b82a4e6",
            "914cb3d8c1944f87a5d532de19766fa5",
            "c8dffcfd0f5046a486324bcfda1045d1",
            "fd101a946184452fbac090de11a5875d",
            "fd8fb341e7174e4bb5b172c867f62fd3"
          ]
        },
        "outputId": "72d07d95-8607-4235-9017-94f0c3eb5d92"
      },
      "execution_count": 20,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/trl/trainer/dpo_trainer.py:300: UserWarning: `max_length` is not set in the DPOTrainer's init it will default to `512` by default, but you should do it yourself in the future.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/trl/trainer/dpo_trainer.py:307: UserWarning: `max_prompt_length` is not set in the DPOTrainer's init it will default to `128` by default, but you should do it yourself in the future.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/trl/trainer/dpo_trainer.py:332: UserWarning: When using DPODataCollatorWithPadding, you should set `remove_unused_columns=False` in your TrainingArguments we have set it for you, but you should do it yourself in the future.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "870e76e22db24d87a915ec1892e13d9e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Map:   0%|          | 0/507 [00:00<?, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "max_steps is given, it will override any value given in num_train_epochs\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:192: UserWarning: Could not find a config file in /scratch/ax2119/huggingFace/Mistral7B - will assume that the vocabulary was not modified.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
            "  warnings.warn(\n",
            "Could not estimate the number of tokens of the input, floating-point operations will not be computed\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='2262' max='40000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [ 2262/40000 2:16:15 < 37:55:24, 0.28 it/s, Epoch 0.97/18]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Rewards/chosen</th>\n",
              "      <th>Rewards/rejected</th>\n",
              "      <th>Rewards/accuracies</th>\n",
              "      <th>Rewards/margins</th>\n",
              "      <th>Logps/rejected</th>\n",
              "      <th>Logps/chosen</th>\n",
              "      <th>Logits/rejected</th>\n",
              "      <th>Logits/chosen</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.624400</td>\n",
              "      <td>0.739023</td>\n",
              "      <td>-2.507345</td>\n",
              "      <td>-3.006997</td>\n",
              "      <td>0.619792</td>\n",
              "      <td>0.499651</td>\n",
              "      <td>-133.772903</td>\n",
              "      <td>-96.816803</td>\n",
              "      <td>-2.277401</td>\n",
              "      <td>-2.278570</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.855700</td>\n",
              "      <td>0.731471</td>\n",
              "      <td>-3.202358</td>\n",
              "      <td>-3.569708</td>\n",
              "      <td>0.559896</td>\n",
              "      <td>0.367350</td>\n",
              "      <td>-139.400024</td>\n",
              "      <td>-103.766922</td>\n",
              "      <td>-2.406795</td>\n",
              "      <td>-2.407934</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.947200</td>\n",
              "      <td>0.842130</td>\n",
              "      <td>-1.359963</td>\n",
              "      <td>-1.491957</td>\n",
              "      <td>0.508464</td>\n",
              "      <td>0.131994</td>\n",
              "      <td>-118.622498</td>\n",
              "      <td>-85.342972</td>\n",
              "      <td>-2.283716</td>\n",
              "      <td>-2.278452</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.750400</td>\n",
              "      <td>0.707543</td>\n",
              "      <td>-1.632179</td>\n",
              "      <td>-2.417706</td>\n",
              "      <td>0.627604</td>\n",
              "      <td>0.785527</td>\n",
              "      <td>-127.879997</td>\n",
              "      <td>-88.065132</td>\n",
              "      <td>-2.312949</td>\n",
              "      <td>-2.309261</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:192: UserWarning: Could not find a config file in /scratch/ax2119/huggingFace/Mistral7B - will assume that the vocabulary was not modified.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:192: UserWarning: Could not find a config file in /scratch/ax2119/huggingFace/Mistral7B - will assume that the vocabulary was not modified.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:192: UserWarning: Could not find a config file in /scratch/ax2119/huggingFace/Mistral7B - will assume that the vocabulary was not modified.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='2356' max='40000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [ 2356/40000 2:21:17 < 37:39:31, 0.28 it/s, Epoch 1.01/18]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Rewards/chosen</th>\n",
              "      <th>Rewards/rejected</th>\n",
              "      <th>Rewards/accuracies</th>\n",
              "      <th>Rewards/margins</th>\n",
              "      <th>Logps/rejected</th>\n",
              "      <th>Logps/chosen</th>\n",
              "      <th>Logits/rejected</th>\n",
              "      <th>Logits/chosen</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.624400</td>\n",
              "      <td>0.739023</td>\n",
              "      <td>-2.507345</td>\n",
              "      <td>-3.006997</td>\n",
              "      <td>0.619792</td>\n",
              "      <td>0.499651</td>\n",
              "      <td>-133.772903</td>\n",
              "      <td>-96.816803</td>\n",
              "      <td>-2.277401</td>\n",
              "      <td>-2.278570</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.855700</td>\n",
              "      <td>0.731471</td>\n",
              "      <td>-3.202358</td>\n",
              "      <td>-3.569708</td>\n",
              "      <td>0.559896</td>\n",
              "      <td>0.367350</td>\n",
              "      <td>-139.400024</td>\n",
              "      <td>-103.766922</td>\n",
              "      <td>-2.406795</td>\n",
              "      <td>-2.407934</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.947200</td>\n",
              "      <td>0.842130</td>\n",
              "      <td>-1.359963</td>\n",
              "      <td>-1.491957</td>\n",
              "      <td>0.508464</td>\n",
              "      <td>0.131994</td>\n",
              "      <td>-118.622498</td>\n",
              "      <td>-85.342972</td>\n",
              "      <td>-2.283716</td>\n",
              "      <td>-2.278452</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.750400</td>\n",
              "      <td>0.707543</td>\n",
              "      <td>-1.632179</td>\n",
              "      <td>-2.417706</td>\n",
              "      <td>0.627604</td>\n",
              "      <td>0.785527</td>\n",
              "      <td>-127.879997</td>\n",
              "      <td>-88.065132</td>\n",
              "      <td>-2.312949</td>\n",
              "      <td>-2.309261</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-20-426ea797cd3c>\u001b[0m in \u001b[0;36m<cell line: 20>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m )\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0mdpo_trainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/trainer.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   1873\u001b[0m                 \u001b[0mhf_hub_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable_progress_bars\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1874\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1875\u001b[0;31m             return inner_training_loop(\n\u001b[0m\u001b[1;32m   1876\u001b[0m                 \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1877\u001b[0m                 \u001b[0mresume_from_checkpoint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mresume_from_checkpoint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/trainer.py\u001b[0m in \u001b[0;36m_inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   2204\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2205\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccelerator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccumulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2206\u001b[0;31m                     \u001b[0mtr_loss_step\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2207\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2208\u001b[0m                 if (\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/trainer.py\u001b[0m in \u001b[0;36mtraining_step\u001b[0;34m(self, model, inputs)\u001b[0m\n\u001b[1;32m   3191\u001b[0m                 \u001b[0mscaled_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3192\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3193\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccelerator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3194\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3195\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgradient_accumulation_steps\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, loss, **kwargs)\u001b[0m\n\u001b[1;32m   2106\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2107\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscaler\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2108\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscaler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscale\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2109\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2110\u001b[0m             \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/_tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    520\u001b[0m                 \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    521\u001b[0m             )\n\u001b[0;32m--> 522\u001b[0;31m         torch.autograd.backward(\n\u001b[0m\u001b[1;32m    523\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    524\u001b[0m         )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    264\u001b[0m     \u001b[0;31m# some Python versions print out the first line of a multi-line function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    265\u001b[0m     \u001b[0;31m# calls in the traceback and some print out the last line\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 266\u001b[0;31m     Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n\u001b[0m\u001b[1;32m    267\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m         \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dpo_trainer.model.save_pretrained(\"/content/drive/MyDrive/ConceptualGapsProject/DPOcheckpoints/MistralDPORedditSumm_3_Finetuned\")"
      ],
      "metadata": {
        "id": "E6u5RbsyFjuf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Checking Model Performance"
      ],
      "metadata": {
        "id": "Q-AEdmVsE6IM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Dataset\n",
        "from datasets import load_dataset\n",
        "from trl import SFTTrainer\n",
        "\n",
        "print(\"Loading Dataset\\n\")\n",
        "\n",
        "# load train jsonl dataset\n",
        "train_dataset = load_dataset(\"json\", data_files=\"/content/drive/MyDrive/ConceptualGapsProject/redditDataset/sftDataset/train.jsonl\", split=\"train\")\n",
        "# load valid jsonl dataset\n",
        "valid_dataset = load_dataset(\"json\", data_files=\"/content/drive/MyDrive/ConceptualGapsProject/redditDataset/sftDataset/valid.jsonl\", split=\"train\")\n",
        "# load test jsonl dataset\n",
        "test_dataset = load_dataset(\"json\", data_files=\"/content/drive/MyDrive/ConceptualGapsProject/redditDataset/sftDataset/test.jsonl\", split=\"train\")\n",
        "\n",
        "def formatting_prompts_func_2(example):\n",
        "  bos_token = \"<s>\"\n",
        "  eos_token = \"</s>\"\n",
        "\n",
        "  full_prompt = \"\"\n",
        "  full_prompt += bos_token\n",
        "  full_prompt += \"### Instruction:\"\n",
        "  full_prompt += \"\\n\" + \"Summarize the given reddit post\"\n",
        "  full_prompt += \"\\n\\n### Input:\"\n",
        "  full_prompt += \"\\n\" + f\"SUBREDDIT: r/{example['subreddit']}\\nTITLE: {example['title']}\\nPOST: {example['post']}\"\n",
        "  full_prompt += \"\\n\\n### Response:\"\n",
        "  # full_prompt += \"\\n\" + f\"Summary: {example['summary']}\"\n",
        "  # full_prompt += eos_token\n",
        "\n",
        "  return full_prompt\n",
        "\n",
        "def formatting_prompts_func_3(example):\n",
        "    bos_token = \"<s>\"\n",
        "    eos_token = \"</s>\"\n",
        "    text = \"\"\n",
        "    text += bos_token\n",
        "    text += f\"SUBREDDIT: r/{example['subreddit']}\\nTITLE: {example['title']}\\nPOST: {example['post']}\\nTL;DR:\" # {example['summary']}\"\n",
        "    # text += eos_token\n",
        "    return text"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 148,
          "referenced_widgets": [
            "2a45f4c99dc04903bb06dd4e1a8c5f26",
            "0e3bdb262a904040a08b803241eed129",
            "d29fae3058994ce5a02bf93c7c6b5f26",
            "347056a5202f48deb349bef84163cb95",
            "ff46e0588bc04c23a66ce0cd3a459c0f",
            "90a0bf1c376f487a971abe06725e46fc",
            "f0285810217f449cbcda96ad2a0eb711",
            "f66ed8eb22b644bf91341b5447eaaaa9",
            "754b5a8bf9b04b2d94e60f870a356da2",
            "90b01f27139e40dc927fdf6e5aa342e1",
            "ae0c58cbba624797a443905563e080f7",
            "9739e2e1aea745b69a83254cd5f2a523",
            "aeb693382ec549e4bca658ec73783dda",
            "1a2fb9f460814f8eac60ec125cc0c1a6",
            "43a1f20d4da64b9991ea936c355a2769",
            "f4ae9cf0dbcf443e8ed64b41a0fe7e16",
            "277dcc376f39405cbdb6946c55142dc3",
            "95ff5a9715da4c97a6f377ba21b75ee0",
            "c6cced4fcdbe483da6b2af9a24ce737b",
            "53eac8c3218c48fd8822152bf0d23bc6",
            "aacc0f85db1a40b4a1a436ee38f3fa0c",
            "ff2e25cf34ec4076b745d4c5676c9ada",
            "bc62027581c94c408f1d4a9b68f5f2ea",
            "2b61dc787b9d4779871d39d24a5ca4c4",
            "724f638e42dd478b8850973b55671979",
            "9fc79394b83d4db29a19d556c96be232",
            "82ccd0e808cc47e181d4e580e666aaf8",
            "38e982c8bdf4489897208cda3c732425",
            "4bc6641da2bd4c0896c02f27d0afdee8",
            "73f76ac4cfa64e188bb5ca9a3d29862a",
            "0f27bc4875474e8fa0c7943c96b507e1",
            "8633ec1c24e8436c92eb6968ae7bec52",
            "da3a94d572ac4466a52ba48cbcc8f88c"
          ]
        },
        "id": "kMDFRc-V0CF9",
        "outputId": "6ea74e58-6f82-44b2-e3c5-c79b0080c35b"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading Dataset\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split: 0 examples [00:00, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2a45f4c99dc04903bb06dd4e1a8c5f26"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split: 0 examples [00:00, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9739e2e1aea745b69a83254cd5f2a523"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split: 0 examples [00:00, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bc62027581c94c408f1d4a9b68f5f2ea"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Loading Model\\n\")\n",
        "\n",
        "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
        "import torch\n",
        "\n",
        "nf4_config = BitsAndBytesConfig(\n",
        "   load_in_4bit=True,\n",
        "   llm_int8_threshold=6.0,\n",
        "   llm_int8_has_fp16_weight=False,\n",
        "   bnb_4bit_quant_type=\"nf4\",\n",
        "   bnb_4bit_use_double_quant=True,\n",
        "   bnb_4bit_compute_dtype=torch.float16\n",
        ")\n",
        "\n",
        "model = AutoModelForCausalLM.from_pretrained(\n",
        "    # \"/content/drive/MyDrive/ConceptualGapsProject/huggingFace\",\n",
        "    \"mistralai/Mistral-7B-v0.1\",\n",
        "    cache_dir = \"/content/mistral_weights/\",\n",
        "    # load_in_4bit=True,\n",
        "    device_map='auto',\n",
        "    quantization_config=nf4_config,\n",
        "    # use_cache=True,\n",
        "    torch_dtype=torch.float16,\n",
        ")\n",
        "model.config.use_cache = False"
      ],
      "metadata": {
        "id": "Sc268HjHlUlW",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84,
          "referenced_widgets": [
            "a09ccf7f81264ca1ab4f502610759c62",
            "b56ba1cee0e245ba9b8241517ff70730",
            "ea5f77c99c5f4004826e57afba3b64cb",
            "30e761b00c6a4a039f3aa12deba101f5",
            "747a565cb17f4f1cadce50174b2391db",
            "f015a1e9ceba4179b7250013363a8ad0",
            "fab4d4526c0c41c4b16074b238085101",
            "16c4e31234ee4d1dbc2b5a4f4d6c8426",
            "dbbc406cddc048a788fed80141bd5ad8",
            "ca79d08daf174ceab5760242d3e0fe65",
            "36d133c170f842d8a27f99f642d593d0"
          ]
        },
        "outputId": "fa6f4b3c-6610-4188-94cd-1067521a1950"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading Model\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a09ccf7f81264ca1ab4f502610759c62"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Tokenizer\n",
        "print(\"Loading Tokenizer\\n\")\n",
        "\n",
        "from transformers import AutoTokenizer\n",
        "\n",
        "# tokenizer = AutoTokenizer.from_pretrained(\"/content/drive/MyDrive/ConceptualGapsProject/huggingFace\", trust_remote_code=True,)\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"mistralai/Mistral-7B-v0.1\")\n",
        "\n",
        "tokenizer.pad_token = tokenizer.eos_token\n",
        "tokenizer.padding_side = \"left\""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CRV6OdFXGW8T",
        "outputId": "3f28c5b5-6aad-4082-ea0d-ebfd1070f7d1"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading Tokenizer\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Loading PEFT Model\n",
        "from peft import PeftModel\n",
        "\n",
        "dpo_model = PeftModel.from_pretrained(model, \"/content/drive/MyDrive/ConceptualGapsProject/DPOcheckpoints/MistralDPORedditSumm_3/checkpoint-3500/train_1\")"
      ],
      "metadata": {
        "id": "5pobFcz3GiS1"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Trying out queries\n",
        "\n",
        "def respond(query):\n",
        "    # eval_prompt_2 = formatting_prompts_func_2(query)\n",
        "    eval_prompt_3 = formatting_prompts_func_3(query)\n",
        "\n",
        "    # model_input_2 = tokenizer(eval_prompt_2, return_tensors=\"pt\").to(\"cuda\")\n",
        "    model_input_3 = tokenizer(eval_prompt_3, return_tensors=\"pt\").to(\"cuda\")\n",
        "\n",
        "    # output_2 = tokenizer.decode(sft_model_2.generate(**model_input_2, max_new_tokens=256, repetition_penalty=1.15)[0], skip_special_tokens=True)\n",
        "    # output_2 = output_2.replace(eval_prompt_2, \"\")\n",
        "\n",
        "    output_3 = tokenizer.decode(dpo_model.generate(**model_input_3, max_new_tokens=256, repetition_penalty=1.15)[0], skip_special_tokens=True)\n",
        "    output_3 = output_3.replace(eval_prompt_3, \"\")\n",
        "\n",
        "    return output_3\n",
        "\n",
        "for i in range(10):\n",
        "    sft_output_3 = respond(test_dataset[i])\n",
        "    print(\"\\n###############\")\n",
        "    # print(\"sft_output_2 : \", sft_output_2)\n",
        "    print(\"\\nsft_output_3 : \", sft_output_3)\n",
        "    print(\"\\nReal Summary : \", test_dataset[i]['summary'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HA9Xpp1Hz79M",
        "outputId": "6f026aa1-ef39-4704-dd82-65fc89d02a58"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "###############\n",
            "\n",
            "sft_output_3 :  SUBREDDIT: r/relationships\n",
            "TITLE: Me [19 F] with my friend [19 M], not sure if I may have messed things up already.\n",
            "POST: Hello hello everybody. I hope this isn't too trivial of a question to ask on here, but I've been feeling a bit out of my depth when it comes to this situation (I've had only one relationship before, and for many reasons, it was out of the ordinary).\n",
            "\n",
            "Okay! So, a couple of weeks ago, I started talking to this guy on Facebook, through a student group that we were both part of. I thought he was sort of cute, so I sent him a PM just to talk, etc, etc. We're both transfer students at the same school, so I knew that we could eventually meet in person once we both moved on-campus. So, we did, and we hung out maybe twice, just as friends.\n",
            "\n",
            "Okay. So, everything is going pretty well. We talk over Facebook and Snapchat, whatever. So, Saturday night, I was just hanging out with people and kind of being bored, when I got a Snapchat from him asking what I was doing. I asked if he wanted to hang out, so we did. \n",
            "\n",
            "We ended up smoking pot (the first time for me, ever), and sort of just wandering around. Eventually we ended up back at his dorm room, where high me decided to just go for it, and I came on to him pretty strongly. It worked out for me (luckily, otherwise things would have been really super awkward), and we ended up messing around but not having sex.\n",
            "\n",
            "Yesterday, however, I ended up going to hang out with him again, and this time we did sleep together. Afterward, we kind of discussed what we were going to do, and he just said that he wanted to \"play it by ear\" and not slap any labels on anything. I'm wondering if this means that he wants a fwb-type situation, or if he might actually be interested in me. The way I've been acting is extremely out of character for me, and I am not interested in having a fuck buddy. I like him, and I would be very interested in maybe seeing where things go, but I'm worried that I may have ruined my chances of a relationship by sleeping with him already.\n",
            "TL;DR: I slept with new acquaintance after making moves on him, and now I'm not sure if he wants a fling or something more. Help? What should I do? Should I wait and see if he contacts me again, or try to discuss our status? Or does it even matter whether he wants more than a hookup? If he doesn't, then I don't want to keep pursuing anything further.\n",
            "\n",
            "Real Summary :  I really like this guy, but after having sex with him after only knowing him for a very brief period of time, I am worried that I may have ruined my chances of a relationship with him.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "###############\n",
            "\n",
            "sft_output_3 :  SUBREDDIT: r/Parenting\n",
            "TITLE: My 11 year old sons friend died suddenly, his funeral is today and my son suddenly doesn't want to attend.\n",
            "POST: **repost from relationships**\n",
            "\n",
            "A couple of weeks ago my sons friend died in a freak accident, it was completely shocking and horrific. He isn't aware of the details, but we broke the news to him as soon as we found out and have spoke about it many times with him.\n",
            "\n",
            "He has cried about it, asked questions and spoken about it with his older siblings (who have also recently lost a friend) and seemed to be okay with it (considering the circumstances).\n",
            "\n",
            "Leading up to the funeral, we have talked about it and explained what he is to expect, etc. This is his first funeral, so we have made sure that he is aware of everything.\n",
            "\n",
            "But today is the day, and he has broken down in the morning and says he doesn't want to go. I have no idea what to do. Do I push him to go? I am worried that he will regret it later, as he is a kid who doesn't like to do new things. But at the same time, I don't want to pressure him to do something he doesn't want to do.\n",
            "Advice, please?\n",
            "TL;DR: Son's friends dies in freak accident, we have been talking about the funeral and he has decided not to go this morning. What should I do? Should I make him? Or let him stay home?\n",
            "\n",
            "Real Summary :  Sons good friend died and his funeral is today. Son was always wanting to go until today. Do I push him to go, or let him skip it?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "###############\n",
            "\n",
            "sft_output_3 :  SUBREDDIT: r/relationships\n",
            "TITLE: The girl [26 F] I [22 M] have been seeing for a month didn't respond to me at all yesterday while hanging out with a friend [~30? M].\n",
            "POST: She gets terrible service while at her house, but I texted her 3 times yesterday, 4-5 hours apart. She didn't call me until early this morning and left a voicemail that she was busy all day with a friend who showed up out of the blue.\n",
            "\n",
            "I saw that she posted a picture of the two of them out of her dead zone house on facebook before I texted her the last time.\n",
            "\n",
            "I don't mind that she hangs out with friends, and I know it's pretty early in the relationship, but am I wrong to be a little annoyed that she didn't respond until 24 hours after my first text?\n",
            "TL;DR: Girl I've been seeing (26) didn't respond to my texts (22M), and I found out later from Facebook that she was hanging out with another person instead of responding to me. Am I being unreasonable to feel a bit annoyed by this?\n",
            "\n",
            "Real Summary :  Girl I'm seeing didn't respond to my texts while hanging out with a friend, what should I do, if anything?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "###############\n",
            "\n",
            "sft_output_3 :  SUBREDDIT: r/tifu\n",
            "TITLE: TIFU by accidently kicking an old woman\n",
            "POST: So this didn't happen today but actually about a year or two ago.\n",
            "\n",
            "I was at my granddads funeral so of course it was all very sad and full of lots of crying old people. After the ceremony everyone walks outside the building and onto the other side of the small road the hearses drive down. Now the road is important because obviously if there's a road, there's a curb onto the sidewalk, so most of us are on the other side of the road, besides a few older people walking a lot slower. \n",
            "\n",
            "As one of the old woman goes to walk up the curb she trips (obviously didn't notice there was one due to crying and whatnot) and I'm the only one who not only sees it coming but is in any position to do anything. So of course as someone who is an avid football (soccer if you're American) player my first instinct is to stick my foot out and kind of control her head like you would with a football.\n",
            "\n",
            "Of course you can imagine this looked horrendously bad on my part (quite literally kicking an old woman while she's down) and as she got up everyone noticed that her nose was completely grey/black as if she'd just been punched in the face. She assures us she's fine and we go to the dinner afterwards where someone finally informs her of her bruising. She goes to the toilet and comes out with a completely normal looking nose so of course everyone's wondering how and it turns out that the \"bruising\" was actually shoe polish from my shoe, confirming to everyone that i kicked this poor old lady square in the face as she fell.\n",
            "TL;DR: Old woman falls over at funeral, I kick her in the face trying to stop her head from bouncing off the ground, she has shoe polish on her nose making everyone think I gave her a black eye.\n",
            "\n",
            "Real Summary :  Tried to stop an old lady falling, kicked her in the face, made her look like she'd been 3 rounds with Mike Tyson\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "###############\n",
            "\n",
            "sft_output_3 :  SUBREDDIT: r/relationships\n",
            "TITLE: I [32 M] found out my wife [31 F] had an affair a year and a half ago, now I can't stop thinking about other women.\n",
            "POST: Wife and I moved accross the country with our two daughters <our dream.  \n",
            "\n",
            "We settle in, happy, rosey veiws, living the dream or so I though.  6mo later she sleeps with a co-worker. I find out about it four months later on my birthday. Shit hits the fan at home and work - she and d-bag are fired. Did I mention she work for a Christian organization?\n",
            "\n",
            "Fast forward 18 months - today. I don't have a marrage - rather - a roomate who's the mother of my children.  She's too fragile to say I want out, but I want out.\n",
            "\n",
            "Do I pursue an \"affair\" or just flip the world upside down my kids and leave?\n",
            "TL;DR: My wife cheated on me, we live together but not married, do I seek comfort from another woman or pack up and move out? (kids are involved)\n",
            "\n",
            "Real Summary :  Wife Cheats on me but I stuck around for kids.  I want out - do I have an affair or break it off and ruin my kids world?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "###############\n",
            "\n",
            "sft_output_3 :  SUBREDDIT: r/loseit\n",
            "TITLE: Caffeine addiction. Anyone taking vitamins or supplements with caffeine?\n",
            "POST: I have a horrible caffeine addiction, and I don't like sacrificing any of my daily calories for coffee. I used to drink 5-6 Diet Dr. Peppers a day, but I have switched to almost exclusively drinking only water most days. I do have a Diet Dr. Pepper occasionally, and I know that technically it's okay to probably have one a day, but I honestly just feel better all around when I drink only water. But if I don't have some caffeine in the morning, I will be writhing in the floor in pain with a skull-pounding headache by mid-day. I know. Ridiculous, right? \n",
            "\n",
            "So, I went to the vitamin aisle the other day in search of a supplement with a little caffeine and the only thing I found was One A Day Adult Vitacraves Gummies with Energy Support. Here's the website with [nutritional info] One serving is 20 calories and has 90 mg of caffeine. There's also some vitamin B6, B12, and folic acid in there that supposedly helps turn food into energy. I have no idea if that's true. I've been taking them a couple days, and so far I do feel more energized and I haven't had any headaches. The issue is that one bottle is around $9 and there are only 60 in a bottle. You have to take 4 (4!) a day, which is very intriguing considering the brand name and all. So it'll cost me around $20 a month. Which is admittedly cheaper than what I used to spend on Diet Dr. Peppers, but still. I was just curious if anyone else had a better option for caffeine in a supplement or vitamin?\n",
            "TL;DR: I have a caffeine addiction and am looking for a vitamin or supplement with caffeine that isn't super expensive. I'm currently using One A Day Adult Vitacraves Gummies with Energy Support, but they are pricey at $20 a month. Any suggestions?\n",
            "\n",
            "Real Summary :  The vitamin with caffeine I'm taking is on the expensive side, and I'm a self-proclaimed tightwad. Anyone have better options?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "###############\n",
            "\n",
            "sft_output_3 :  SUBREDDIT: r/tifu\n",
            "TITLE: TIFU by eating a banana.\n",
            "POST: So, this actually happened yesterday. I am currently studying abroad in a tropical country. As a result, the selection of locally grown fruit is phenomenal. Now, I'm not usually one for bananas, but I keeping hearing they are much more flavorful here, so I decided I'd give it a go and bought a small bunch from the local market. Yesterday morning, I decided I'd have one for breakfast along with my usual corn flakes. After I wash it down with coffee, I immediately start feeling like crap, but one of my roommates had a stomach bug over the weekend, so I just automatically assume that I must have contracted said bug and rush off to my four hour class. Four hours of hell, that is. As soon as I sit down, my nose becomes a waterfall with off and on sneezing fits. Next, cue the most god awful indigestion and acid reflux I have ever experienced. I keep telling myself that I only have to make it to 1 o' clock, then I can just go to bed. Easier said than done, but I finally get home and find little success in napping. Once I get up, I discover that I have a nice little rash starting in each armpit, which I chalk up to having switched deodorants just that morning. Finish off a bottle of Maalox with some advil and commenced sleep attempt number two. I wake up a little more clear headed this time and begin to assess my situation. The rash is now worse and I start wondering, what if the deodorant didn't cause the rash? Is there a single thing that could possibly explain all of my symptoms? I started to ask myself out loud, \"Did I eat anything today that I don't usually..... oh, shit.\" In an instant my fingers were flying on Google to search \"banana allergy\" only to find a multitude of symptom lists that pretty much specifically described my day of hell. By this time, my face had even started to swell a bit, but now knowing what I  was up against, I bought some benedryl, had a couple of beers and went back to sleep. Woke up this morning feeling much better, and unlike Pam Poovey, I will not continue to stuff my face with foods I know I am allergic to. No more bananas. Ever.\n",
            "TL;DR: Eat banana for breakfast, feel like death, google symptoms, realize I am probably allergic to bananas. Not going near them again. Ever.\n",
            "TL PROBABLE SYMPTOMS OF BANANA ALLERGY: Indigestion, Acid Reflux, Nausea, Vomiting, Rash, Swelling, Sneezing Fits, Runny Nose, Watery Eyes, Itchiness, Wheezing, Difficult Breathing. (I had almost all of these.)\n",
            "\n",
            "Real Summary :  I don't usually eat bananas, but decided to have one for breakfast since I'm studying abroad in a tropical country and the bananas are supposedly better here, unaware that I am apparently highly allergic. Paid an accordingly hefty price.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "###############\n",
            "\n",
            "sft_output_3 :  SUBREDDIT: r/relationships\n",
            "TITLE: Me [16F] have been dating my bf [16M] for 2 years and have not done anything sexually yet, is this alright?\n",
            "POST: Hi guy's I just need some advice to calm my nerves. \n",
            "\n",
            "So I have been dating this guy for 2 years since we were the age of 14. The most we have done is make out and cuddle, that's it nothing more. \n",
            "\n",
            "We are only 16 and I was wondering is it a bad thing we have not done anything sexually for the 2 years we have been dating? because you hear of people taking it slow, but are we going to slow?\n",
            "\n",
            "I have not spoken to him about this, but I just need an outside opinion on this before I do. Because I don't want to mess this up, I really love this guy\n",
            "TL;DR: me (16) and boyfriend (16) have been dating for two years and have only made out and cuddled, is this okay or should i talk to him about doing something sexual?\n",
            "\n",
            "Real Summary :  Have been dating my Bf since the age of 14, we have not done anything except make out and cuddle. Is this alright or does it hurt the relationship long term?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "###############\n",
            "\n",
            "sft_output_3 :  SUBREDDIT: r/AskReddit\n",
            "TITLE: My fellow redditors! Within reason.. turn your ridiculous ideas into porn!\n",
            "POST: Soo there isn't a question in the headline.. How about.. Reddit, what kind of porn would you make if you could?\n",
            "\n",
            "Be respectful. I may be a whore, but I'm a human being above all else!\n",
            "\n",
            "I am in the process of developing a website for my alter ego porn star. \n",
            "\n",
            "I've decided that aside from all the normal boring porn stuff that I want really memorable pornos. Full length movies or short clips.\n",
            "\n",
            "So far we are having a scenario where I am dared to suck off a corpse in a morgue.\n",
            "\n",
            "I'm also going to fuck every super hero to which a costume is available and I am particularly excited about that!\n",
            "\n",
            "Giant mud pit. Need I say more?\n",
            "\n",
            "Sadly, unicorn porn was already made...\n",
            "\n",
            "So, in this vast pool of insane and colorful minds that is reddit.. I ask you to share with me elaborate or not so elaborate ideas for porn. They don't all have to be silly, some things you legitimately might want to see and never do or never see enough of it.\n",
            "\n",
            "In all seriousness your idea could get turned into the real thing and if it does I will send you a free copy of the production so you can show your friends!\n",
            "TL;DR: I need ideas for porn scenarios to produce as full length videos or short clips. If yours gets used I will send you a free copy of the finished product!\n",
            "\n",
            "Real Summary :  If you can think of a good idea(retarded OR reasonable) for a film on my site and we film it, you will get a free DVD copy.\n",
            "\n",
            "###############\n",
            "\n",
            "sft_output_3 :  SUBREDDIT: r/relationships\n",
            "TITLE: Me [18M] questioning if I should ask out my friend [17F]\n",
            "POST: Ok so perhaps this isn't the most dramatic thing you have seen on this subreddit but I feel like I'm stuck.\n",
            "\n",
            "I have been friends with this girl for a while now asking her and things were going great until I asked her to prom. I am hoping this will be a great night however ever since I asked her a couple weeks ago I find I can no longer talk to her. Out conversations are awkward with me trying to start the majority of topics (Badly written but you get the picture)\n",
            "\n",
            "I really like the girl but i'm wondering if she saw too much into the prom-posal. Should I just give up, or ask her out? Im not sure\n",
            "TL;DR: Asked girl to prom, conversation has become awkward, unsure if I should keep pursuing relationship or back off. Help Reddit!\n",
            "TL dont want to make her uncomfortable by continuing to try and talk to her if she is not interested in anything more than friendship. However I do like her and would like to continue talking to her and see where it goes.\n",
            "\n",
            "Real Summary :  Asked a girl (Friend) I like to prom and now our interactions are awkward. To ask her out or not to ask her out?\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "kSuFwqPBIbHu",
        "outputId": "e95190ca-836d-4af4-f283-9859b26c0cb7"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'I really like this guy, but after having sex with him after only knowing him for a very brief period of time, I am worried that I may have ruined my chances of a relationship with him.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "JAee-dKXIwER"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}