From 6506076a881012a5b406abb3bb45ded293c9cb1f Mon Sep 17 00:00:00 2001 From: Amy Roberts <22614925+amyeroberts@users.noreply.github.com> Date: Mon, 8 Jan 2024 16:09:09 +0000 Subject: [PATCH 1/6] Update meatdata loading for oneformer --- .../oneformer/image_processing_oneformer.py | 15 +++++-- .../test_image_processing_oneformer.py | 41 +++++++++++-------- 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/src/transformers/models/oneformer/image_processing_oneformer.py b/src/transformers/models/oneformer/image_processing_oneformer.py index c42001a96252f2..e52dad20cb9efa 100644 --- a/src/transformers/models/oneformer/image_processing_oneformer.py +++ b/src/transformers/models/oneformer/image_processing_oneformer.py @@ -15,6 +15,7 @@ """Image processor class for OneFormer.""" import json +import os import warnings from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union @@ -332,8 +333,15 @@ def get_oneformer_resize_output_image_size( def prepare_metadata(repo_path, class_info_file): - with open(hf_hub_download(repo_path, class_info_file, repo_type="dataset"), "r") as f: + fname = os.path.join("" if repo_path is None else repo_path, class_info_file) + + if not os.path.exists(fname) or not os.path.isfile(fname): + # File cannot be found locally, try downloading from hub + fname = hf_hub_download(repo_path, class_info_file, repo_type="dataset") + + with open(fname, "r") as f: class_info = json.load(f) + metadata = {} class_names = [] thing_ids = [] @@ -387,7 +395,8 @@ class OneFormerImageProcessor(BaseImageProcessor): is used for background, and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by `ignore_index`. repo_path (`str`, defaults to `shi-labs/oneformer_demo`, *optional*, defaults to `"shi-labs/oneformer_demo"`): - Dataset repository on huggingface hub containing the JSON file with class information for the dataset. + Path to dataset repo or local directory containing the JSON file with class information for the dataset. + If unset, will load `class_info_file` from the current working directory. class_info_file (`str`, *optional*): JSON file containing class information for the dataset. It is stored inside on the `repo_path` dataset repository. @@ -409,7 +418,7 @@ def __init__( image_std: Union[float, List[float]] = None, ignore_index: Optional[int] = None, do_reduce_labels: bool = False, - repo_path: str = "shi-labs/oneformer_demo", + repo_path: Optional[str] = "shi-labs/oneformer_demo", class_info_file: str = None, num_text: Optional[int] = None, **kwargs, diff --git a/tests/models/oneformer/test_image_processing_oneformer.py b/tests/models/oneformer/test_image_processing_oneformer.py index 6fa95f2341477c..9b9f7691836059 100644 --- a/tests/models/oneformer/test_image_processing_oneformer.py +++ b/tests/models/oneformer/test_image_processing_oneformer.py @@ -15,10 +15,10 @@ import json +import tempfile import unittest import numpy as np -from huggingface_hub import hf_hub_download from transformers.testing_utils import require_torch, require_vision from transformers.utils import is_torch_available, is_vision_available @@ -31,29 +31,13 @@ if is_vision_available(): from transformers import OneFormerImageProcessor - from transformers.models.oneformer.image_processing_oneformer import binary_mask_to_rle + from transformers.models.oneformer.image_processing_oneformer import binary_mask_to_rle, prepare_metadata from transformers.models.oneformer.modeling_oneformer import OneFormerForUniversalSegmentationOutput if is_vision_available(): from PIL import Image -def prepare_metadata(class_info_file, repo_path="shi-labs/oneformer_demo"): - with open(hf_hub_download(repo_path, class_info_file, repo_type="dataset"), "r") as f: - class_info = json.load(f) - metadata = {} - class_names = [] - thing_ids = [] - for key, info in class_info.items(): - metadata[key] = info["name"] - class_names.append(info["name"]) - if info["isthing"]: - thing_ids.append(int(key)) - metadata["thing_ids"] = thing_ids - metadata["class_names"] = class_names - return metadata - - class OneFormerImageProcessorTester(unittest.TestCase): def __init__( self, @@ -332,3 +316,24 @@ def test_post_process_panoptic_segmentation(self): self.assertEqual( el["segmentation"].shape, (self.image_processor_tester.height, self.image_processor_tester.width) ) + + def test_can_load_with_local_metadata(self): + # Create a temporary json file + metadata = { + "0": {"isthing": 0, "name": "foo"}, + "1": {"isthing": 0, "name": "bar"}, + "2": {"isthing": 1, "name": "baz"}, + } + + with tempfile.TemporaryDirectory() as tmpdirname: + metadata_path = os.path.join(tmpdirname, "metadata.json") + with open(metadata_path, "w") as f: + json.dump(metadata, f) + + config_dict = self.image_processor_dict() + config_dict["class_info_file"] = metadata_path + config_dict["repo_path"] = tmpdirname + + image_processor = self.image_processing_class(**config_dict) + + self.assertEqual(image_processor.metadata, metadata) From 9fabd834a29e59fc7d1446a0fc464f0b0776d0e6 Mon Sep 17 00:00:00 2001 From: Amy Roberts <22614925+amyeroberts@users.noreply.github.com> Date: Mon, 8 Jan 2024 16:19:20 +0000 Subject: [PATCH 2/6] Enable loading from a model repo --- .../models/oneformer/image_processing_oneformer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/oneformer/image_processing_oneformer.py b/src/transformers/models/oneformer/image_processing_oneformer.py index e52dad20cb9efa..00d2e35da02358 100644 --- a/src/transformers/models/oneformer/image_processing_oneformer.py +++ b/src/transformers/models/oneformer/image_processing_oneformer.py @@ -21,6 +21,7 @@ import numpy as np from huggingface_hub import hf_hub_download +from huggingface_hub.utils import RepositoryNotFoundError from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict from ...image_transforms import ( @@ -336,8 +337,11 @@ def prepare_metadata(repo_path, class_info_file): fname = os.path.join("" if repo_path is None else repo_path, class_info_file) if not os.path.exists(fname) or not os.path.isfile(fname): - # File cannot be found locally, try downloading from hub - fname = hf_hub_download(repo_path, class_info_file, repo_type="dataset") + # We try downloading from a dataset by default for backward compatibility + try: + fname = hf_hub_download(repo_path, class_info_file, repo_type="dataset") + except RepositoryNotFoundError: + fname = hf_hub_download(repo_path, class_info_file) with open(fname, "r") as f: class_info = json.load(f) From 1af694b0d04d16218f99bcc5f881393c69c45ff0 Mon Sep 17 00:00:00 2001 From: Amy Roberts <22614925+amyeroberts@users.noreply.github.com> Date: Mon, 8 Jan 2024 16:31:55 +0000 Subject: [PATCH 3/6] Update docstrings --- .../oneformer/image_processing_oneformer.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/oneformer/image_processing_oneformer.py b/src/transformers/models/oneformer/image_processing_oneformer.py index 00d2e35da02358..3fba6e007027eb 100644 --- a/src/transformers/models/oneformer/image_processing_oneformer.py +++ b/src/transformers/models/oneformer/image_processing_oneformer.py @@ -333,15 +333,15 @@ def get_oneformer_resize_output_image_size( return output_size -def prepare_metadata(repo_path, class_info_file): - fname = os.path.join("" if repo_path is None else repo_path, class_info_file) +def prepare_metadata(repo_id, class_info_file): + fname = os.path.join("" if repo_id is None else repo_id, class_info_file) if not os.path.exists(fname) or not os.path.isfile(fname): # We try downloading from a dataset by default for backward compatibility try: - fname = hf_hub_download(repo_path, class_info_file, repo_type="dataset") + fname = hf_hub_download(repo_id, class_info_file, repo_type="dataset") except RepositoryNotFoundError: - fname = hf_hub_download(repo_path, class_info_file) + fname = hf_hub_download(repo_id, class_info_file) with open(fname, "r") as f: class_info = json.load(f) @@ -398,12 +398,11 @@ class OneFormerImageProcessor(BaseImageProcessor): Whether or not to decrement all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by `ignore_index`. - repo_path (`str`, defaults to `shi-labs/oneformer_demo`, *optional*, defaults to `"shi-labs/oneformer_demo"`): - Path to dataset repo or local directory containing the JSON file with class information for the dataset. + repo_path (`str`, *optional*, defaults to `"shi-labs/oneformer_demo"`): + Path to hub repo or local directory containing the JSON file with class information for the dataset. If unset, will load `class_info_file` from the current working directory. class_info_file (`str`, *optional*): - JSON file containing class information for the dataset. It is stored inside on the `repo_path` dataset - repository. + JSON file containing class information for the dataset. See `shi-labs/oneformer_demo/cityscapes_panoptic.json` for an example. num_text (`int`, *optional*): Number of text entries in the text input list. """ @@ -443,6 +442,9 @@ def __init__( ) do_reduce_labels = kwargs.pop("reduce_labels") + if class_info_file is None: + raise ValueError("You must provide a `class_info_file`") + super().__init__(**kwargs) self.do_resize = do_resize self.size = size From f2b2e45dd5895c23640f7ab19cbe4480760f1a9d Mon Sep 17 00:00:00 2001 From: Amy Roberts <22614925+amyeroberts@users.noreply.github.com> Date: Mon, 8 Jan 2024 18:50:57 +0000 Subject: [PATCH 4/6] Fix tests --- tests/models/oneformer/test_image_processing_oneformer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/models/oneformer/test_image_processing_oneformer.py b/tests/models/oneformer/test_image_processing_oneformer.py index 9b9f7691836059..56a9c91814d5e5 100644 --- a/tests/models/oneformer/test_image_processing_oneformer.py +++ b/tests/models/oneformer/test_image_processing_oneformer.py @@ -15,6 +15,7 @@ import json +import os import tempfile import unittest @@ -69,7 +70,7 @@ def __init__( self.image_mean = image_mean self.image_std = image_std self.class_info_file = class_info_file - self.metadata = prepare_metadata(class_info_file, repo_path) + self.metadata = prepare_metadata(repo_path, class_info_file) self.num_text = num_text self.repo_path = repo_path From 0779f421fca83baa3bfade8140aa2701c1dd172c Mon Sep 17 00:00:00 2001 From: Amy Roberts <22614925+amyeroberts@users.noreply.github.com> Date: Wed, 10 Jan 2024 15:08:25 +0000 Subject: [PATCH 5/6] Update tests --- .../oneformer/image_processing_oneformer.py | 30 +++++++++++-------- .../test_image_processing_oneformer.py | 10 +++---- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/transformers/models/oneformer/image_processing_oneformer.py b/src/transformers/models/oneformer/image_processing_oneformer.py index 3fba6e007027eb..7b236c0a14c945 100644 --- a/src/transformers/models/oneformer/image_processing_oneformer.py +++ b/src/transformers/models/oneformer/image_processing_oneformer.py @@ -333,7 +333,21 @@ def get_oneformer_resize_output_image_size( return output_size -def prepare_metadata(repo_id, class_info_file): +def prepare_metadata(class_info): + metadata = {} + class_names = [] + thing_ids = [] + for key, info in class_info.items(): + metadata[key] = info["name"] + class_names.append(info["name"]) + if info["isthing"]: + thing_ids.append(int(key)) + metadata["thing_ids"] = thing_ids + metadata["class_names"] = class_names + return metadata + + +def load_metadata(repo_id, class_info_file): fname = os.path.join("" if repo_id is None else repo_id, class_info_file) if not os.path.exists(fname) or not os.path.isfile(fname): @@ -346,17 +360,7 @@ def prepare_metadata(repo_id, class_info_file): with open(fname, "r") as f: class_info = json.load(f) - metadata = {} - class_names = [] - thing_ids = [] - for key, info in class_info.items(): - metadata[key] = info["name"] - class_names.append(info["name"]) - if info["isthing"]: - thing_ids.append(int(key)) - metadata["thing_ids"] = thing_ids - metadata["class_names"] = class_names - return metadata + return class_info class OneFormerImageProcessor(BaseImageProcessor): @@ -458,7 +462,7 @@ def __init__( self.do_reduce_labels = do_reduce_labels self.class_info_file = class_info_file self.repo_path = repo_path - self.metadata = prepare_metadata(repo_path, class_info_file) + self.metadata = prepare_metadata(load_metadata(repo_path, class_info_file)) self.num_text = num_text def resize( diff --git a/tests/models/oneformer/test_image_processing_oneformer.py b/tests/models/oneformer/test_image_processing_oneformer.py index 56a9c91814d5e5..4a9e560463adf0 100644 --- a/tests/models/oneformer/test_image_processing_oneformer.py +++ b/tests/models/oneformer/test_image_processing_oneformer.py @@ -70,7 +70,6 @@ def __init__( self.image_mean = image_mean self.image_std = image_std self.class_info_file = class_info_file - self.metadata = prepare_metadata(repo_path, class_info_file) self.num_text = num_text self.repo_path = repo_path @@ -95,7 +94,6 @@ def prepare_image_processor_dict(self): "do_reduce_labels": self.do_reduce_labels, "ignore_index": self.ignore_index, "class_info_file": self.class_info_file, - "metadata": self.metadata, "num_text": self.num_text, } @@ -320,21 +318,21 @@ def test_post_process_panoptic_segmentation(self): def test_can_load_with_local_metadata(self): # Create a temporary json file - metadata = { + class_info = { "0": {"isthing": 0, "name": "foo"}, "1": {"isthing": 0, "name": "bar"}, "2": {"isthing": 1, "name": "baz"}, } + metadata = prepare_metadata(class_info) with tempfile.TemporaryDirectory() as tmpdirname: metadata_path = os.path.join(tmpdirname, "metadata.json") with open(metadata_path, "w") as f: - json.dump(metadata, f) + json.dump(class_info, f) - config_dict = self.image_processor_dict() + config_dict = self.image_processor_dict config_dict["class_info_file"] = metadata_path config_dict["repo_path"] = tmpdirname - image_processor = self.image_processing_class(**config_dict) self.assertEqual(image_processor.metadata, metadata) From a77766833b865cd4b0b5d3c3c9ac74174c168bd6 Mon Sep 17 00:00:00 2001 From: Amy Roberts <22614925+amyeroberts@users.noreply.github.com> Date: Thu, 11 Jan 2024 15:16:19 +0000 Subject: [PATCH 6/6] Clarify repo_path behaviour --- .../models/oneformer/image_processing_oneformer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/oneformer/image_processing_oneformer.py b/src/transformers/models/oneformer/image_processing_oneformer.py index 7b236c0a14c945..385124d1b995ba 100644 --- a/src/transformers/models/oneformer/image_processing_oneformer.py +++ b/src/transformers/models/oneformer/image_processing_oneformer.py @@ -351,6 +351,8 @@ def load_metadata(repo_id, class_info_file): fname = os.path.join("" if repo_id is None else repo_id, class_info_file) if not os.path.exists(fname) or not os.path.isfile(fname): + if repo_id is None: + raise ValueError(f"Could not file {fname} locally. repo_id must be defined if loading from the hub") # We try downloading from a dataset by default for backward compatibility try: fname = hf_hub_download(repo_id, class_info_file, repo_type="dataset") @@ -404,7 +406,7 @@ class OneFormerImageProcessor(BaseImageProcessor): The background label will be replaced by `ignore_index`. repo_path (`str`, *optional*, defaults to `"shi-labs/oneformer_demo"`): Path to hub repo or local directory containing the JSON file with class information for the dataset. - If unset, will load `class_info_file` from the current working directory. + If unset, will look for `class_info_file` in the current working directory. class_info_file (`str`, *optional*): JSON file containing class information for the dataset. See `shi-labs/oneformer_demo/cityscapes_panoptic.json` for an example. num_text (`int`, *optional*):