PruneDziribert.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/MostHumble/ModelPruning/blob/main/PruneDziribert.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9CW2h9-FJHYo"
      },
      "source": [
        "# Load the old tokenizer, save it locally"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%pip install transformers"
      ],
      "metadata": {
        "id": "wYYoThv9JjKn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "WQIHw16wJHYp",
        "outputId": "ae5f871a-5407-4f61-9ad6-8d74a7a84c1c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 130,
          "referenced_widgets": [
            "b0fe95d65eeb4a27a84016b606df9958",
            "bf4ed50efae441cab8b6690a2a2abc51",
            "c8e1ed1b795947bd911c74f0009c8d9f",
            "b8da49bf888f4723bb6579f5281ce210",
            "3ecfe7f60e244d8fa62d3f0d413986f3",
            "fd004d8511cd465bb02813147def4f28",
            "252eaace24f64168b0a3c5f1eb8f5b14",
            "f43ac7bf3c0e45a194e57fe4eb1d77a8",
            "cee6d3c9bd2045a7873e6e18c719adc9",
            "a601e1ee42ec4cfe80d518ef3d0c5bc0",
            "90f7027132cf42d1ba325d4921c5d52c",
            "7bdb189d919f43fdbde77e36aa78bcf8",
            "b7bd8c323aca4ad880ec0409bb6debea",
            "c1ae5a36605c4ab7962f8dd1d8d8eaf8",
            "da5069a1f31a4de69646c19b470ec930",
            "f0712d681885452186a20a8ab3db38d3",
            "8f9f9b0e57bd4f3793f6672dedfe8dc3",
            "8b71a69d2b5e47d0a9841e789a61acfb",
            "683bf1b4e09a45489e76a02df1b00b08",
            "14e3128b338843eb9dc1b2f61c78c936",
            "c697b475be754f4ba1c0311cad94b9b6",
            "61a71d86b1434c5f84642023bf24003d",
            "1346795327844832b8278e672109754c",
            "ee21f574d52f42ffaf8c564208a337d8",
            "7b9718c5d2fd44d594db9490221596ad",
            "7d846f1763ff422f8cd3d28e74830a0c",
            "dcd368c667014bd69b25b22a396fa6a2",
            "20fb14fdb41e4d78ab0d6346101aa104",
            "7270dae6cc8041c4ad5f68eec11e21c4",
            "5060b3975b3545cb877b277c1833c16d",
            "3e2589827dbc4208828d678fabcb0d7d",
            "cb667c3ea5e945d8a84cc59d9b561e68",
            "7fdd56b66c44482684dd956bab8270cd"
          ]
        }
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/176 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b0fe95d65eeb4a27a84016b606df9958"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/620 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7bdb189d919f43fdbde77e36aa78bcf8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/446k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1346795327844832b8278e672109754c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "50000\n"
          ]
        }
      ],
      "source": [
        "from transformers import AutoTokenizer\n",
        "old_tok_path = \"/content/ModelPruning/old_tokenizer\"\n",
        "\n",
        "model_name = \"alger-ia/dziribert\"\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "\n",
        "tokenizer.save_pretrained(old_tok_path)\n",
        "\n",
        "print(len(tokenizer))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D2dd8K1tJHYr"
      },
      "source": [
        "# Save the tokenizer to edit"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EoVRmjjEJHYr"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "\n",
        "tokenizer_file = old_tok_path + \"/tokenizer.json\"\n",
        "with open(tokenizer_file, \"r\") as f:\n",
        "  data = json.load(f)\n",
        "\n",
        "print(data[\"model\"][\"vocab\"])\n",
        "print(f\"length of tokenizer: {len(tokenizer)}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ntRdAJ7uJHYs"
      },
      "source": [
        "# Find unwanted tokens & get rid of them\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "stinky_tokens =  {'ぁ': 414, 'あ': 415, 'い': 416, 'う': 417, 'ぇ': 418, 'え': 419, 'お': 420, 'か': 421, 'き': 422, 'く': 423, 'け': 424, 'こ': 425, 'さ': 426, 'し': 427, 'す': 428, 'せ': 429, 'そ': 430, 'た': 431, 'ち': 432, 'っ': 433, 'つ': 434, 'て': 435, 'と': 436, 'な': 437, 'に': 438, 'ね': 439, 'の': 440, 'は': 441, 'ひ': 442, 'ふ': 443, 'へ': 444, 'ほ': 445, 'ま': 446, 'み': 447, 'む': 448, 'め': 449, 'も': 450, 'ゃ': 451, 'や': 452, 'ゅ': 453, 'ゆ': 454, 'ょ': 455, 'よ': 456, 'ら': 457, 'り': 458, 'る': 459, 'れ': 460, 'ろ': 461, 'わ': 462, 'を': 463, 'ん': 464, 'ァ': 465, 'ア': 466, 'ィ': 467, 'イ': 468, 'ウ': 469, 'ェ': 470, 'エ': 471, 'ォ': 472, 'オ': 473, 'カ': 474, 'キ': 475, 'ク': 476, 'ケ': 477, 'コ': 478, 'サ': 479, 'シ': 480, 'ス': 481, 'セ': 482, 'ソ': 483, 'タ': 484, 'チ': 485, 'ッ': 486, 'ツ': 487, 'テ': 488, 'ト': 489, 'ナ': 490, 'ニ': 491, 'ネ': 492, 'ノ': 493, 'ハ': 494, 'ヒ': 495, 'フ': 496, 'ヘ': 497, 'ホ': 498, 'マ': 499, 'ミ': 500, 'ム': 501, 'メ': 502, 'モ': 503, 'ャ': 504, 'ヤ': 505, 'ュ': 506, 'ョ': 507, 'ラ': 508, 'リ': 509, 'ル': 510, 'レ': 511, 'ロ': 512, 'ワ': 513, 'ン': 514, '・': 515, 'ー': 516, 'ヽ': 517, 'ㅋ': 518, 'ㅠ': 519, 'ㅤ': 520, '一': 521, '上': 522, '下': 523, '中': 524, '事': 525, '人': 526, '今': 527, '休': 528, '会': 529, '体': 530, '何': 531, '作': 532, '使': 533, '信': 534, '先': 535, '入': 536, '全': 537, '円': 538, '写': 539, '出': 540, '分': 541, '初': 542, '前': 543, '力': 544, '加': 545, '動': 546, '勝': 547, '募': 548, '参': 549, '取': 550, '可': 551, '合': 552, '同': 553, '名': 554, '味': 555, '品': 556, '問': 557, '回': 558, '場': 559, '変': 560, '多': 561, '大': 562, '天': 563, '女': 564, '好': 565, '嬉': 566, '子': 567, '定': 568, '実': 569, '家': 570, '対': 571, '小': 572, '少': 573, '年': 574, '店': 575, '度': 576, '張': 577, '強': 578, '当': 579, '待': 580, '後': 581, '心': 582, '応': 583, '思': 584, '意': 585, '愛': 586, '感': 587, '戦': 588, '手': 589, '持': 590, '新': 591, '方': 592, '日': 593, '明': 594, '時': 595, '更': 596, '最': 597, '月': 598, '本': 599, '来': 600, '業': 601, '楽': 602, '様': 603, '欲': 604, '気': 605, '無': 606, '物': 607, '理': 608, '生': 609, '田': 610, '画': 611, '番': 612, '発': 613, '的': 614, '目': 615, '真': 616, '知': 617, '私': 618, '笑': 619, '素': 620, '結': 621, '美': 622, '者': 623, '自': 624, '良': 625, '色': 626, '行': 627, '見': 628, '言': 629, '話': 630, '買': 631, '質': 632, '近': 633, '通': 634, '違': 635, '部': 636, '野': 637, '金': 638, '開': 639, '間': 640, '関': 641, '集': 642, '面': 643, '頑': 644, '願': 645, '食': 646, '高': 647, 'अ': 225, 'आ': 226, 'इ': 227, 'ई': 228, 'उ': 229, 'ए': 230, 'औ': 231, 'क': 232, 'ख': 233, 'ग': 234, 'च': 235, 'छ': 236, 'ज': 237, 'ट': 238, 'ड': 239, 'ण': 240, 'त': 241, 'थ': 242, 'द': 243, 'ध': 244, 'न': 245, 'प': 246, 'फ': 247, 'ब': 248, 'भ': 249, 'म': 250, 'य': 251, 'र': 252, 'ल': 253, 'व': 254, 'श': 255, 'ष': 256, 'स': 257, 'ह': 258, 'ा': 259, 'ि': 260, 'ी': 261, 'ो': 262, '।': 263, 'ক': 264, 'ত': 265, 'ন': 266, 'ব': 267, 'ম': 268, 'য': 269, 'র': 270, 'া': 271, 'ি': 272, 'ে': 273, 'ર': 274, 'ા': 275, 'କ': 276, 'ତ': 277, 'ନ': 278, 'ବ': 279, 'ମ': 280, 'ର': 281, 'ା': 282, 'େ': 283, 'ก': 284, 'ค': 285, 'ง': 286, 'ด': 287, 'ต': 288, 'น': 289, 'บ': 290, 'ม': 291, 'ย': 292, 'ร': 293, 'ล': 294, 'ว': 295, 'ห': 296, 'อ': 297, 'ะ': 298, 'า': 299, 'เ': 300, 'แ': 301, 'ไ': 302, 'ღ': 303, 'ᄀ': 304, 'ᄁ': 305, 'ᄂ': 306, 'ᄃ': 307, 'ᄅ': 308, 'ᄆ': 309, 'ᄇ': 310, 'ᄈ': 311, 'ᄉ': 312, 'ᄋ': 313, 'ᄌ': 314, 'ᄎ': 315, 'ᄏ': 316, 'ᄐ': 317, 'ᄑ': 318, 'ᄒ': 319, 'ᅡ': 320, 'ᅢ': 321, 'ᅣ': 322, 'ᅥ': 323, 'ᅦ': 324, 'ᅧ': 325, 'ᅩ': 326, 'ᅪ': 327, 'ᅬ': 328, 'ᅭ': 329, 'ᅮ': 330, 'ᅲ': 331, 'ᅳ': 332, 'ᅴ': 333, 'ᅵ': 334, 'ᆨ': 335, 'ᆫ': 336, 'ᆮ': 337, 'ᆯ': 338, 'ᆷ': 339, 'ᆸ': 340, 'ᆺ': 341, 'ᆻ': 342, 'ᆼ': 343, 'ᆾ': 344, 'ᇀ': 345, '¿': 81, '×': 82, 'ß': 83, 'æ': 84, 'ð': 85, 'ø': 86, 'đ': 87, 'ı': 88, 'ł': 89, 'ŋ': 90, 'œ': 91, 'ə': 92, 'ɞ': 93, 'ʚ': 94, 'α': 95, 'ε': 96, 'ι': 97, 'ο': 98, 'ω': 99, 'а': 100, 'б': 101, 'в': 102, 'г': 103, 'д': 104, 'е': 105, 'ж': 106, 'з': 107, 'и': 108, 'к': 109, 'л': 110, 'м': 111, 'н': 112, 'о': 113, 'п': 114, 'р': 115, 'с': 116, 'т': 117, 'у': 118, 'ф': 119, 'х': 120, 'ц': 121, 'ч': 122, 'ш': 123, 'щ': 124, 'ы': 125, 'ь': 126, 'э': 127, 'ю': 128, 'я': 129, 'є': 130, 'і': 131, 'א': 132, 'ב': 133, 'ד': 134, 'ה': 135, 'ו': 136, 'י': 137, 'ל': 138, 'ם': 139, 'מ': 140, 'נ': 141, 'ע': 142, 'ר': 143, 'ש': 144, 'ת': 145}\n",
        "\n",
        "# I don't believe that '<S>', '<T>' behave like special tokens here (check special_tokens_map.json)\n",
        "special_tokens = {'[PAD]': 0,'[UNK]': 1,'[CLS]': 2,'[SEP]': 3,'[MASK]': 4,'<S>': 5,'<T>': 6}"
      ],
      "metadata": {
        "id": "NUf7bZAagOc2"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "\n",
        "def is_wanted(token):\n",
        "    # Check if the token contains alphabet characters (a-zA-Z)\n",
        "    if bool(re.search(r'[a-zA-Z]', token)):\n",
        "        return False\n",
        "    # Check if the token is not in stinky_tokens\n",
        "    if token in stinky_tokens:\n",
        "        return False\n",
        "    return True\n",
        "\n",
        "wanted_tokens = {}\n",
        "\n",
        "unwanted_tokens = {}\n",
        "\n",
        "vocabs = data[\"model\"][\"vocab\"]\n",
        "\n",
        "k = 0\n",
        "\n",
        "l = len(special_tokens)\n",
        "\n",
        "wanted_index_list = [i for i in range(k)]\n",
        "unwanted_index_list = []\n",
        "\n",
        "for vocab, j in vocabs.items():\n",
        "    if is_wanted(vocab) or j<l:\n",
        "        wanted_tokens[vocab] = k\n",
        "        k+=1\n",
        "        wanted_index_list.append(j)\n",
        "    else:\n",
        "        unwanted_tokens[vocab] = j\n",
        "        unwanted_index_list.append(j)\n",
        "\n",
        "unwanted_tokens.update(stinky_tokens)\n",
        "\n",
        "\n",
        "print(len(wanted_tokens))  # outputs the number of \"wanted\" tokens\n",
        "print(len(unwanted_tokens))  # outputs the number of \"unwanted\" tokens\n",
        "\n",
        "assert len(tokenizer) == len(wanted_tokens)+len(unwanted_tokens)"
      ],
      "metadata": {
        "id": "ta5XL5Guag1P",
        "outputId": "79e4d5b9-7b40-4233-be11-f82d4ab1f827",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "34643\n",
            "15357\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "nmZyWGKGJHYv",
        "outputId": "10e843bb-4f44-4bb1-8331-8d893fcd52af",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "34643"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ],
      "source": [
        "from copy import deepcopy\n",
        "\n",
        "data[\"model\"][\"vocab\"] = deepcopy(wanted_tokens)\n",
        "len(data[\"model\"][\"vocab\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jHN4yaitJHYz"
      },
      "source": [
        "# Save new tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from distutils.dir_util import copy_tree\n",
        "\n",
        "new_tok_path = \"ModelPruning/new_tokenizer\"\n",
        "!mkdir {new_tok_path}\n",
        "copy_tree(old_tok_path, new_tok_path)"
      ],
      "metadata": {
        "id": "aKVCae0pmMVm",
        "outputId": "1f2cc6b8-2d5a-40c3-9cae-f564fe85f25b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['ModelPruning/new_tokenizer/tokenizer.json',\n",
              " 'ModelPruning/new_tokenizer/special_tokens_map.json',\n",
              " 'ModelPruning/new_tokenizer/tokenizer_config.json',\n",
              " 'ModelPruning/new_tokenizer/vocab.txt']"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "zklGfbgIJHYz"
      },
      "outputs": [],
      "source": [
        "with open(new_tok_path+\"/tokenizer.json\", 'w') as f:\n",
        "    json.dump(data,f)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IwQcvRDBJHY1"
      },
      "source": [
        "# Add/Remove embeddings from word embedding matrix in model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7YV2HnUtJHY1"
      },
      "source": [
        "This is necessary because word embeddings are being deleted.\n",
        "\n",
        "e.g. what used to be 50000 is now 34643"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "m_knATVjJHY1",
        "outputId": "a5fc1274-56ee-4770-aecf-1486080c4e1c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "length of modified tokenizer: 34643\n"
          ]
        }
      ],
      "source": [
        "from transformers import BertTokenizerFast,BertForMaskedLM\n",
        "tokenizer_modified  = BertTokenizerFast.from_pretrained(new_tok_path)\n",
        "print(f\"length of modified tokenizer: {len(tokenizer_modified)}\")\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer_modified.push_to_hub(\"Sifal/DzarbiBert\",commit_message=\"1st\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eh9PYmSyxcAd",
        "outputId": "13e00035-613a-48eb-c4ab-0f8e36da805e"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "CommitInfo(commit_url='https://huggingface.co/Sifal/DzarbiBert/commit/fee305f02a30a4985af5b23f0530590f07617c41', commit_message='1st', commit_description='', oid='fee305f02a30a4985af5b23f0530590f07617c41', pr_url=None, pr_revision=None, pr_num=None)"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "oe7GCuqiJHY1",
        "outputId": "c6d94bd2-ea4d-4953-e8d7-b95ad7b104f7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "e57a92e1b8354395bf21c2e1cffaa39d",
            "a7ec564559354bc1b6802bb39bc55999",
            "0eeff36dd8144c1faf9a45d226cdfdcd",
            "b2619c286e9b46ec989dc2bc002ff8e4",
            "38d6e1a4bf914aaa999a7f74dab29e76",
            "d4c7c48c6f4343a9b366ef2cf750dd5f",
            "a19defb7553b404cb295674fae76cdb6",
            "70be94724c3e4304975fe0caa04f3ef5",
            "ac4e7ca2fe394d6eb78467e1c1b861a4",
            "08ee2ca592f84c31b42c1324c499cd68",
            "86937b159a7d4e028e137e7bce52ae23"
          ]
        }
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading model.safetensors:   0%|          | 0.00/498M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e57a92e1b8354395bf21c2e1cffaa39d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "474.97mb\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "import os\n",
        "\n",
        "model = BertForMaskedLM.from_pretrained(model_name)\n",
        "model.save_pretrained(\"./original_model/\")\n",
        "print(f'{os.path.getsize(\"./original_model/pytorch_model.bin\") / (1024 ** 2):.2f}mb')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# check the original model word embedding shape and type\n",
        "original_embedding = model.bert.embeddings.word_embeddings.weight\n",
        "print(original_embedding.shape) # torch.Size([50000, 768])\n",
        "print(type(original_embedding)) # <class 'torch.nn.parameter.Parameter'>"
      ],
      "metadata": {
        "id": "LGntRkYpR4Hi",
        "outputId": "c7e71b4a-d77d-441b-ee1a-45d1a5055447",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([50000, 768])\n",
            "<class 'torch.nn.parameter.Parameter'>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# select only the rows of embeddings we want\n",
        "row_select = torch.tensor(wanted_index_list)\n",
        "selected_embedding = torch.index_select(original_embedding, 0, row_select)\n",
        "\n",
        "# resize model embeddings\n",
        "model.resize_token_embeddings(len(wanted_index_list))\n",
        "print(f\"shape after resizing embeddings: {selected_embedding.shape}\")\n",
        "\n",
        "# Making the embeddings a multiple of 16 It can provide a minor speedup when training or doing inference.\n",
        "# https://twitter.com/karpathy/status/1621578354024677377\n",
        "\n",
        "while selected_embedding.shape[0] % 64 != 0:\n",
        "    selected_embedding = torch.vstack([selected_embedding, torch.zeros(1, selected_embedding.shape[1])])\n",
        "print(f\"shape after ceiling embeddings to mod 64: {selected_embedding.shape}\")\n",
        "\n",
        "\n",
        "# assign the selected embeddings to be the new model embeddings\n",
        "model.bert.embeddings.word_embeddings.weight.data = torch.nn.parameter.Parameter(selected_embedding)\n",
        "model.cls.predictions.decoder.bias.data =  torch.nn.parameter.Parameter(torch.zeros(selected_embedding.shape[0]),requires_grad=True)\n",
        "model.cls.predictions.bias = torch.nn.parameter.Parameter(torch.zeros(selected_embedding.shape[0]),requires_grad=True)\n",
        "# check the new embedding shape\n",
        "print(f\"shape of model embeddings :{model.bert.embeddings.word_embeddings.weight.shape}\")\n",
        "model.cls.predictions.decoder.bias\n",
        "\n",
        "# update model config\n",
        "model.config.update({\n",
        "    \"vocab_size\": selected_embedding.shape[0]\n",
        "})"
      ],
      "metadata": {
        "id": "EdVSkTIUQtBy",
        "outputId": "f5f8258c-90b2-4062-d46a-01c4319dfd12",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape after resizing embeddings: torch.Size([34643, 768])\n",
            "shape after ceiling embeddings to mod 64: torch.Size([34688, 768])\n",
            "shape of model embeddings :torch.Size([34688, 768])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "So this is like a ~30.62% size decrease in embedding size"
      ],
      "metadata": {
        "id": "HPjOkZEdbmMB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model.save_pretrained(\"./modified_model/\")\n",
        "print(f'{os.path.getsize(\"./modified_model/pytorch_model.bin\") / (1024 ** 2):.2f}mb')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aC8NHoq5az3o",
        "outputId": "9dd811b1-2c95-4e79-9347-81896716be80"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "430.18mb\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.push_to_hub(\"Sifal/DzarbiBert\",commit_message=\"model\")"
      ],
      "metadata": {
        "id": "-sRMgxI7oAVG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t_n2b3mrJHZE"
      },
      "source": [
        "### Making sure  both the embeddings and tokenizer are working correctly"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "kXOND231JHZE",
        "outputId": "96a915bf-9028-4514-eeb7-8162566f6f3f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[2, 173, 3]\n",
            "[2, 82, 3]\n"
          ]
        }
      ],
      "source": [
        "old_ids = tokenizer(\"ك\").input_ids\n",
        "print(old_ids)\n",
        "\n",
        "new_ids = tokenizer_modified(\"ك\").input_ids\n",
        "print(new_ids)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "lIKBCU9GJHZE",
        "outputId": "2054a2e6-5224-4d85-a3d8-a246ce07cb0f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(True)"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ],
      "source": [
        "import torch\n",
        "\n",
        "old_embeddings = torch.stack([original_embedding[i] for i in old_ids])\n",
        "new_embeddings = torch.stack([model.bert.embeddings.word_embeddings.weight.data[i] for i in new_ids])\n",
        "\n",
        "torch.all(torch.eq(old_embeddings, new_embeddings))"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "base",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.10"
    },
    "orig_nbformat": 4,
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b0fe95d65eeb4a27a84016b606df9958": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_bf4ed50efae441cab8b6690a2a2abc51",
              "IPY_MODEL_c8e1ed1b795947bd911c74f0009c8d9f",
              "IPY_MODEL_b8da49bf888f4723bb6579f5281ce210"
            ],
            "layout": "IPY_MODEL_3ecfe7f60e244d8fa62d3f0d413986f3"
          }
        },
        "bf4ed50efae441cab8b6690a2a2abc51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fd004d8511cd465bb02813147def4f28",
            "placeholder": "​",
            "style": "IPY_MODEL_252eaace24f64168b0a3c5f1eb8f5b14",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "c8e1ed1b795947bd911c74f0009c8d9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f43ac7bf3c0e45a194e57fe4eb1d77a8",
            "max": 176,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cee6d3c9bd2045a7873e6e18c719adc9",
            "value": 176
          }
        },
        "b8da49bf888f4723bb6579f5281ce210": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a601e1ee42ec4cfe80d518ef3d0c5bc0",
            "placeholder": "​",
            "style": "IPY_MODEL_90f7027132cf42d1ba325d4921c5d52c",
            "value": " 176/176 [00:00&lt;00:00, 2.25kB/s]"
          }
        },
        "3ecfe7f60e244d8fa62d3f0d413986f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fd004d8511cd465bb02813147def4f28": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "252eaace24f64168b0a3c5f1eb8f5b14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f43ac7bf3c0e45a194e57fe4eb1d77a8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cee6d3c9bd2045a7873e6e18c719adc9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a601e1ee42ec4cfe80d518ef3d0c5bc0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "90f7027132cf42d1ba325d4921c5d52c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7bdb189d919f43fdbde77e36aa78bcf8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b7bd8c323aca4ad880ec0409bb6debea",
              "IPY_MODEL_c1ae5a36605c4ab7962f8dd1d8d8eaf8",
              "IPY_MODEL_da5069a1f31a4de69646c19b470ec930"
            ],
            "layout": "IPY_MODEL_f0712d681885452186a20a8ab3db38d3"
          }
        },
        "b7bd8c323aca4ad880ec0409bb6debea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8f9f9b0e57bd4f3793f6672dedfe8dc3",
            "placeholder": "​",
            "style": "IPY_MODEL_8b71a69d2b5e47d0a9841e789a61acfb",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "c1ae5a36605c4ab7962f8dd1d8d8eaf8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_683bf1b4e09a45489e76a02df1b00b08",
            "max": 620,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_14e3128b338843eb9dc1b2f61c78c936",
            "value": 620
          }
        },
        "da5069a1f31a4de69646c19b470ec930": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c697b475be754f4ba1c0311cad94b9b6",
            "placeholder": "​",
            "style": "IPY_MODEL_61a71d86b1434c5f84642023bf24003d",
            "value": " 620/620 [00:00&lt;00:00, 6.93kB/s]"
          }
        },
        "f0712d681885452186a20a8ab3db38d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8f9f9b0e57bd4f3793f6672dedfe8dc3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8b71a69d2b5e47d0a9841e789a61acfb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "683bf1b4e09a45489e76a02df1b00b08": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "14e3128b338843eb9dc1b2f61c78c936": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c697b475be754f4ba1c0311cad94b9b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "61a71d86b1434c5f84642023bf24003d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1346795327844832b8278e672109754c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ee21f574d52f42ffaf8c564208a337d8",
              "IPY_MODEL_7b9718c5d2fd44d594db9490221596ad",
              "IPY_MODEL_7d846f1763ff422f8cd3d28e74830a0c"
            ],
            "layout": "IPY_MODEL_dcd368c667014bd69b25b22a396fa6a2"
          }
        },
        "ee21f574d52f42ffaf8c564208a337d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_20fb14fdb41e4d78ab0d6346101aa104",
            "placeholder": "​",
            "style": "IPY_MODEL_7270dae6cc8041c4ad5f68eec11e21c4",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "7b9718c5d2fd44d594db9490221596ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5060b3975b3545cb877b277c1833c16d",
            "max": 446200,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3e2589827dbc4208828d678fabcb0d7d",
            "value": 446200
          }
        },
        "7d846f1763ff422f8cd3d28e74830a0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cb667c3ea5e945d8a84cc59d9b561e68",
            "placeholder": "​",
            "style": "IPY_MODEL_7fdd56b66c44482684dd956bab8270cd",
            "value": " 446k/446k [00:00&lt;00:00, 878kB/s]"
          }
        },
        "dcd368c667014bd69b25b22a396fa6a2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "20fb14fdb41e4d78ab0d6346101aa104": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7270dae6cc8041c4ad5f68eec11e21c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5060b3975b3545cb877b277c1833c16d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3e2589827dbc4208828d678fabcb0d7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cb667c3ea5e945d8a84cc59d9b561e68": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7fdd56b66c44482684dd956bab8270cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e57a92e1b8354395bf21c2e1cffaa39d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a7ec564559354bc1b6802bb39bc55999",
              "IPY_MODEL_0eeff36dd8144c1faf9a45d226cdfdcd",
              "IPY_MODEL_b2619c286e9b46ec989dc2bc002ff8e4"
            ],
            "layout": "IPY_MODEL_38d6e1a4bf914aaa999a7f74dab29e76"
          }
        },
        "a7ec564559354bc1b6802bb39bc55999": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d4c7c48c6f4343a9b366ef2cf750dd5f",
            "placeholder": "​",
            "style": "IPY_MODEL_a19defb7553b404cb295674fae76cdb6",
            "value": "Downloading model.safetensors: 100%"
          }
        },
        "0eeff36dd8144c1faf9a45d226cdfdcd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_70be94724c3e4304975fe0caa04f3ef5",
            "max": 497999424,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ac4e7ca2fe394d6eb78467e1c1b861a4",
            "value": 497999424
          }
        },
        "b2619c286e9b46ec989dc2bc002ff8e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_08ee2ca592f84c31b42c1324c499cd68",
            "placeholder": "​",
            "style": "IPY_MODEL_86937b159a7d4e028e137e7bce52ae23",
            "value": " 498M/498M [00:22&lt;00:00, 23.5MB/s]"
          }
        },
        "38d6e1a4bf914aaa999a7f74dab29e76": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d4c7c48c6f4343a9b366ef2cf750dd5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a19defb7553b404cb295674fae76cdb6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "70be94724c3e4304975fe0caa04f3ef5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ac4e7ca2fe394d6eb78467e1c1b861a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "08ee2ca592f84c31b42c1324c499cd68": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "86937b159a7d4e028e137e7bce52ae23": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}