Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add-new-model-like: fix broken script #32460

Closed
wants to merge 4 commits into from
Closed

add-new-model-like: fix broken script #32460

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cbc80c8
fix:show all processor classes including fast image processor
RUFFY-369 Aug 5, 2024
e8efc53
fix:bug causing add_model_like script fail
RUFFY-369 Aug 5, 2024
3919017
Merge branch 'huggingface:main' into fix_add_model
RUFFY-369 Aug 6, 2024
ddbfff2
style:make fixup
RUFFY-369 Aug 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 41 additions & 7 deletions src/transformers/commands/add_new_model_like.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -368,14 +368,29 @@ def replace_model_patterns(

# Now let's replace every other attribute by their placeholder
for attr in attributes_to_check:
text = text.replace(getattr(old_model_patterns, attr), ATTRIBUTE_TO_PLACEHOLDER[attr])
get_attr = (
getattr(old_model_patterns, attr)[0]
if isinstance(getattr(old_model_patterns, attr), tuple)
else getattr(old_model_patterns, attr)
)
text = text.replace(get_attr, ATTRIBUTE_TO_PLACEHOLDER[attr])

# Finally we can replace the placeholder byt the new values.
replacements = []
for attr, placeholder in ATTRIBUTE_TO_PLACEHOLDER.items():
if placeholder in text:
replacements.append((getattr(old_model_patterns, attr), getattr(new_model_patterns, attr)))
text = text.replace(placeholder, getattr(new_model_patterns, attr))
get_attr = (
getattr(old_model_patterns, attr)[0]
if isinstance(getattr(old_model_patterns, attr), tuple)
else getattr(old_model_patterns, attr)
)
get_attr_new = (
getattr(new_model_patterns, attr)[0]
if isinstance(getattr(new_model_patterns, attr), tuple)
else getattr(new_model_patterns, attr)
)
replacements.append((get_attr, get_attr_new))
text = text.replace(placeholder, get_attr_new)

# If we have two inconsistent replacements, we don't return anything (ex: GPT2->GPT_NEW and GPT2->GPTNew)
old_replacement_values = [old for old, new in replacements]
Expand Down Expand Up @@ -530,7 +545,17 @@ def duplicate_module(
special_pattern = False
for pattern, attr in SPECIAL_PATTERNS.items():
if pattern in obj:
obj = obj.replace(getattr(old_model_patterns, attr), getattr(new_model_patterns, attr))
obj_attr = (
getattr(old_model_patterns, attr)[0]
if isinstance(getattr(old_model_patterns, attr), tuple)
else getattr(old_model_patterns, attr)
)
obj_attr_new = (
getattr(new_model_patterns, attr)[0]
if isinstance(getattr(new_model_patterns, attr), tuple)
else getattr(new_model_patterns, attr)
)
obj = obj.replace(obj_attr, obj_attr_new)
new_objects.append(obj)
special_pattern = True
break
Expand Down Expand Up @@ -951,7 +976,9 @@ def add_model_to_main_init(
if not with_processing:
processing_classes = [
old_model_patterns.tokenizer_class,
old_model_patterns.image_processor_class,
old_model_patterns.image_processor_class[0]
if isinstance(old_model_patterns.image_processor_class, tuple)
else old_model_patterns.image_processor_class,
old_model_patterns.feature_extractor_class,
old_model_patterns.processor_class,
]
Expand Down Expand Up @@ -1069,7 +1096,12 @@ def add_model_to_auto_classes(
and new_model_patterns.image_processor_class is not None
):
new_patterns.append(
pattern.replace("{image_processor_class}", old_model_patterns.image_processor_class)
pattern.replace(
"{image_processor_class}",
old_model_patterns.image_processor_class[0]
if isinstance(old_model_patterns.image_processor_class, tuple)
else old_model_patterns.image_processor_class,
)
)
elif "{feature_extractor_class}" in pattern:
if (
Expand Down Expand Up @@ -1628,9 +1660,11 @@ def get_user_input():
)

old_processing_classes = [
c if not isinstance(c, tuple) else c[0]
c_i
for c in [old_image_processor_class, old_feature_extractor_class, old_tokenizer_class, old_processor_class]
if c is not None
for c_i in (c if isinstance(c, tuple) else [c])
if c_i is not None
]
old_processing_classes = ", ".join(old_processing_classes)
keep_processing = get_user_field(
Expand Down