From 560ff4c8a64d50096559b34d3e395597c2ea9233 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iker=20Garc=C3=ADa-Ferrero?= Date: Sat, 23 Dec 2023 14:12:06 +0100 Subject: [PATCH 1/5] Update image_processing_videomae.py Creating a tensor from a list of numpy.ndarrays is extremely slow. This small change makes the preprocessing step much faster. --- src/transformers/models/videomae/image_processing_videomae.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/videomae/image_processing_videomae.py b/src/transformers/models/videomae/image_processing_videomae.py index 6df708eec3ea04..ac53baaa5c01b7 100644 --- a/src/transformers/models/videomae/image_processing_videomae.py +++ b/src/transformers/models/videomae/image_processing_videomae.py @@ -339,5 +339,5 @@ def preprocess( for video in videos ] - data = {"pixel_values": videos} + data = {"pixel_values": np.asarray(videos)} return BatchFeature(data=data, tensor_type=return_tensors) From 6da5b6f8585bfcbcfd43e8e3a9229c94afcf2b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iker=20Garc=C3=ADa-Ferrero?= Date: Thu, 4 Jan 2024 15:46:22 +0100 Subject: [PATCH 2/5] Update src/transformers/models/videomae/image_processing_videomae.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --- src/transformers/models/videomae/image_processing_videomae.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/videomae/image_processing_videomae.py b/src/transformers/models/videomae/image_processing_videomae.py index ac53baaa5c01b7..e32e65ee56370b 100644 --- a/src/transformers/models/videomae/image_processing_videomae.py +++ b/src/transformers/models/videomae/image_processing_videomae.py @@ -339,5 +339,7 @@ def preprocess( for video in videos ] - data = {"pixel_values": np.asarray(videos)} + if return_tensors is not None: + # Speeds up tensor conversion - see: https://github.com/huggingface/transformers/pull/28221/files + data = {"pixel_values": np.asarray(videos)} return BatchFeature(data=data, tensor_type=return_tensors) From 1df5602e57a5bdcaa451e586705ec68481596d61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iker=20Garc=C3=ADa?= Date: Thu, 4 Jan 2024 16:01:30 +0100 Subject: [PATCH 3/5] Simplify the code --- .../models/videomae/image_processing_videomae.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/videomae/image_processing_videomae.py b/src/transformers/models/videomae/image_processing_videomae.py index e32e65ee56370b..5515980cdab7c6 100644 --- a/src/transformers/models/videomae/image_processing_videomae.py +++ b/src/transformers/models/videomae/image_processing_videomae.py @@ -339,7 +339,7 @@ def preprocess( for video in videos ] - if return_tensors is not None: - # Speeds up tensor conversion - see: https://github.com/huggingface/transformers/pull/28221/files - data = {"pixel_values": np.asarray(videos)} + # Speeds up tensor conversion - see: https://github.com/huggingface/transformers/pull/28221/files + data = {"pixel_values": np.asarray(videos) if return_tensors == "pt" else videos} + return BatchFeature(data=data, tensor_type=return_tensors) From 2731b82c8a23a17e76126bf1dcab2a8d5961cf89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iker=20Garc=C3=ADa?= Date: Wed, 31 Jan 2024 20:42:52 +0100 Subject: [PATCH 4/5] Recursive ndarray check --- src/transformers/feature_extraction_utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/transformers/feature_extraction_utils.py b/src/transformers/feature_extraction_utils.py index fe1f7a78c93f74..258039f226ffe8 100644 --- a/src/transformers/feature_extraction_utils.py +++ b/src/transformers/feature_extraction_utils.py @@ -135,8 +135,13 @@ def _get_is_as_tensor_fns(self, tensor_type: Optional[Union[str, TensorType]] = raise ImportError("Unable to convert output to PyTorch tensors format, PyTorch is not installed.") import torch # noqa + def recursive_ndarray_check(value): + if isinstance(value, (list, tuple)) and len(value) > 0: + return recursive_ndarray_check(value[0]) + return isinstance(value, np.ndarray) + def as_tensor(value): - if isinstance(value, (list, tuple)) and len(value) > 0 and isinstance(value[0], np.ndarray): + if recursive_ndarray_check(value): value = np.array(value) return torch.tensor(value) From 9098ad61b7009c2952a1e76b6c83e79ad66c5020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iker=20Garc=C3=ADa-Ferrero?= Date: Fri, 8 Mar 2024 12:32:23 +0100 Subject: [PATCH 5/5] Update src/transformers/models/videomae/image_processing_videomae.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --- src/transformers/models/videomae/image_processing_videomae.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/videomae/image_processing_videomae.py b/src/transformers/models/videomae/image_processing_videomae.py index 5515980cdab7c6..7501668bd853ff 100644 --- a/src/transformers/models/videomae/image_processing_videomae.py +++ b/src/transformers/models/videomae/image_processing_videomae.py @@ -340,6 +340,6 @@ def preprocess( ] # Speeds up tensor conversion - see: https://github.com/huggingface/transformers/pull/28221/files - data = {"pixel_values": np.asarray(videos) if return_tensors == "pt" else videos} + data = {"pixel_values": np.asarray(videos) if return_tensors is not None else videos} return BatchFeature(data=data, tensor_type=return_tensors)