From cbc80c890f858cfa6a571b2ba6730c17e93add02 Mon Sep 17 00:00:00 2001 From: RUFFY-369 Date: Mon, 5 Aug 2024 21:51:56 +0530 Subject: [PATCH 1/3] fix:show all processor classes including fast image processor --- src/transformers/commands/add_new_model_like.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/transformers/commands/add_new_model_like.py b/src/transformers/commands/add_new_model_like.py index e4b2f9be5cf3dc..7e875f2eb1cef8 100644 --- a/src/transformers/commands/add_new_model_like.py +++ b/src/transformers/commands/add_new_model_like.py @@ -1628,9 +1628,10 @@ def get_user_input(): ) old_processing_classes = [ - c if not isinstance(c, tuple) else c[0] - for c in [old_image_processor_class, old_feature_extractor_class, old_tokenizer_class, old_processor_class] + c_i for c in [old_image_processor_class, old_feature_extractor_class, old_tokenizer_class, old_processor_class] if c is not None + for c_i in (c if isinstance(c, tuple) else [c]) + if c_i is not None ] old_processing_classes = ", ".join(old_processing_classes) keep_processing = get_user_field( From e8efc53766593b15f53a7f929edfd41dcd59f1b3 Mon Sep 17 00:00:00 2001 From: RUFFY-369 Date: Mon, 5 Aug 2024 22:02:42 +0530 Subject: [PATCH 2/3] fix:bug causing add_model_like script fail --- .../commands/add_new_model_like.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/transformers/commands/add_new_model_like.py b/src/transformers/commands/add_new_model_like.py index 7e875f2eb1cef8..69824994f05514 100644 --- a/src/transformers/commands/add_new_model_like.py +++ b/src/transformers/commands/add_new_model_like.py @@ -368,14 +368,18 @@ def replace_model_patterns( # Now let's replace every other attribute by their placeholder for attr in attributes_to_check: - text = text.replace(getattr(old_model_patterns, attr), ATTRIBUTE_TO_PLACEHOLDER[attr]) + get_attr = getattr(old_model_patterns, attr)[0] if isinstance(getattr(old_model_patterns, attr), tuple) else getattr(old_model_patterns, attr) + text = text.replace(get_attr, ATTRIBUTE_TO_PLACEHOLDER[attr]) + # Finally we can replace the placeholder byt the new values. replacements = [] for attr, placeholder in ATTRIBUTE_TO_PLACEHOLDER.items(): if placeholder in text: - replacements.append((getattr(old_model_patterns, attr), getattr(new_model_patterns, attr))) - text = text.replace(placeholder, getattr(new_model_patterns, attr)) + get_attr = getattr(old_model_patterns, attr)[0] if isinstance(getattr(old_model_patterns, attr), tuple) else getattr(old_model_patterns, attr) + get_attr_new = getattr(new_model_patterns, attr)[0] if isinstance(getattr(new_model_patterns, attr), tuple) else getattr(new_model_patterns, attr) + replacements.append((get_attr, get_attr_new)) + text = text.replace(placeholder, get_attr_new) # If we have two inconsistent replacements, we don't return anything (ex: GPT2->GPT_NEW and GPT2->GPTNew) old_replacement_values = [old for old, new in replacements] @@ -530,7 +534,9 @@ def duplicate_module( special_pattern = False for pattern, attr in SPECIAL_PATTERNS.items(): if pattern in obj: - obj = obj.replace(getattr(old_model_patterns, attr), getattr(new_model_patterns, attr)) + obj_attr = getattr(old_model_patterns, attr)[0] if isinstance(getattr(old_model_patterns, attr), tuple) else getattr(old_model_patterns, attr) + obj_attr_new = getattr(new_model_patterns, attr)[0] if isinstance(getattr(new_model_patterns, attr), tuple) else getattr(new_model_patterns, attr) + obj = obj.replace(obj_attr, obj_attr_new) new_objects.append(obj) special_pattern = True break @@ -951,7 +957,7 @@ def add_model_to_main_init( if not with_processing: processing_classes = [ old_model_patterns.tokenizer_class, - old_model_patterns.image_processor_class, + old_model_patterns.image_processor_class[0] if isinstance(old_model_patterns.image_processor_class, tuple) else old_model_patterns.image_processor_class, old_model_patterns.feature_extractor_class, old_model_patterns.processor_class, ] @@ -1069,7 +1075,7 @@ def add_model_to_auto_classes( and new_model_patterns.image_processor_class is not None ): new_patterns.append( - pattern.replace("{image_processor_class}", old_model_patterns.image_processor_class) + pattern.replace("{image_processor_class}", old_model_patterns.image_processor_class[0] if isinstance(old_model_patterns.image_processor_class, tuple) else old_model_patterns.image_processor_class) ) elif "{feature_extractor_class}" in pattern: if ( From ddbfff29cb23716f80bd463ed14ed0ac8dd595c7 Mon Sep 17 00:00:00 2001 From: RUFFY-369 Date: Tue, 6 Aug 2024 15:19:40 +0530 Subject: [PATCH 3/3] style:make fixup --- .../commands/add_new_model_like.py | 45 +++++++++++++++---- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/src/transformers/commands/add_new_model_like.py b/src/transformers/commands/add_new_model_like.py index 69824994f05514..1b7b060dba7167 100644 --- a/src/transformers/commands/add_new_model_like.py +++ b/src/transformers/commands/add_new_model_like.py @@ -368,16 +368,27 @@ def replace_model_patterns( # Now let's replace every other attribute by their placeholder for attr in attributes_to_check: - get_attr = getattr(old_model_patterns, attr)[0] if isinstance(getattr(old_model_patterns, attr), tuple) else getattr(old_model_patterns, attr) + get_attr = ( + getattr(old_model_patterns, attr)[0] + if isinstance(getattr(old_model_patterns, attr), tuple) + else getattr(old_model_patterns, attr) + ) text = text.replace(get_attr, ATTRIBUTE_TO_PLACEHOLDER[attr]) - # Finally we can replace the placeholder byt the new values. replacements = [] for attr, placeholder in ATTRIBUTE_TO_PLACEHOLDER.items(): if placeholder in text: - get_attr = getattr(old_model_patterns, attr)[0] if isinstance(getattr(old_model_patterns, attr), tuple) else getattr(old_model_patterns, attr) - get_attr_new = getattr(new_model_patterns, attr)[0] if isinstance(getattr(new_model_patterns, attr), tuple) else getattr(new_model_patterns, attr) + get_attr = ( + getattr(old_model_patterns, attr)[0] + if isinstance(getattr(old_model_patterns, attr), tuple) + else getattr(old_model_patterns, attr) + ) + get_attr_new = ( + getattr(new_model_patterns, attr)[0] + if isinstance(getattr(new_model_patterns, attr), tuple) + else getattr(new_model_patterns, attr) + ) replacements.append((get_attr, get_attr_new)) text = text.replace(placeholder, get_attr_new) @@ -534,8 +545,16 @@ def duplicate_module( special_pattern = False for pattern, attr in SPECIAL_PATTERNS.items(): if pattern in obj: - obj_attr = getattr(old_model_patterns, attr)[0] if isinstance(getattr(old_model_patterns, attr), tuple) else getattr(old_model_patterns, attr) - obj_attr_new = getattr(new_model_patterns, attr)[0] if isinstance(getattr(new_model_patterns, attr), tuple) else getattr(new_model_patterns, attr) + obj_attr = ( + getattr(old_model_patterns, attr)[0] + if isinstance(getattr(old_model_patterns, attr), tuple) + else getattr(old_model_patterns, attr) + ) + obj_attr_new = ( + getattr(new_model_patterns, attr)[0] + if isinstance(getattr(new_model_patterns, attr), tuple) + else getattr(new_model_patterns, attr) + ) obj = obj.replace(obj_attr, obj_attr_new) new_objects.append(obj) special_pattern = True @@ -957,7 +976,9 @@ def add_model_to_main_init( if not with_processing: processing_classes = [ old_model_patterns.tokenizer_class, - old_model_patterns.image_processor_class[0] if isinstance(old_model_patterns.image_processor_class, tuple) else old_model_patterns.image_processor_class, + old_model_patterns.image_processor_class[0] + if isinstance(old_model_patterns.image_processor_class, tuple) + else old_model_patterns.image_processor_class, old_model_patterns.feature_extractor_class, old_model_patterns.processor_class, ] @@ -1075,7 +1096,12 @@ def add_model_to_auto_classes( and new_model_patterns.image_processor_class is not None ): new_patterns.append( - pattern.replace("{image_processor_class}", old_model_patterns.image_processor_class[0] if isinstance(old_model_patterns.image_processor_class, tuple) else old_model_patterns.image_processor_class) + pattern.replace( + "{image_processor_class}", + old_model_patterns.image_processor_class[0] + if isinstance(old_model_patterns.image_processor_class, tuple) + else old_model_patterns.image_processor_class, + ) ) elif "{feature_extractor_class}" in pattern: if ( @@ -1634,7 +1660,8 @@ def get_user_input(): ) old_processing_classes = [ - c_i for c in [old_image_processor_class, old_feature_extractor_class, old_tokenizer_class, old_processor_class] + c_i + for c in [old_image_processor_class, old_feature_extractor_class, old_tokenizer_class, old_processor_class] if c is not None for c_i in (c if isinstance(c, tuple) else [c]) if c_i is not None