From 202ae8cb8aee367708e70eefa4cd1ef4d79a2550 Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Fri, 6 Sep 2024 14:17:31 +0100 Subject: [PATCH 1/5] Added to the class --- darwin/datatypes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/darwin/datatypes.py b/darwin/datatypes.py index 5b79c7958..c8da1322c 100644 --- a/darwin/datatypes.py +++ b/darwin/datatypes.py @@ -422,6 +422,10 @@ class Property: # Description of the property description: Optional[str] = None + # Granularity of the property + # If none, we assume it is a section-level property + granularity: Optional[str] = None + @dataclass class PropertyClass: From f719794aeec54d8462cd32c407225531496f4ef7 Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Thu, 12 Sep 2024 15:48:49 +0100 Subject: [PATCH 2/5] Import of annotation-level properties --- darwin/datatypes.py | 23 +++++++++++++++++----- darwin/future/data_objects/properties.py | 25 ++++++++++++++++++++---- darwin/future/tests/core/fixtures.py | 7 ++++++- darwin/importer/importer.py | 4 ++++ darwin/utils/utils.py | 3 ++- tests/darwin/client_test.py | 1 + 6 files changed, 52 insertions(+), 11 deletions(-) diff --git a/darwin/datatypes.py b/darwin/datatypes.py index c8da1322c..c2ef99d26 100644 --- a/darwin/datatypes.py +++ b/darwin/datatypes.py @@ -24,7 +24,11 @@ except ImportError: NDArray = Any # type:ignore -from darwin.future.data_objects.properties import PropertyType, SelectedProperty +from darwin.future.data_objects.properties import ( + PropertyType, + SelectedProperty, + PropertyGranularity, +) from darwin.path_utils import construct_full_path, is_properties_enabled, parse_metadata # Utility types @@ -423,8 +427,7 @@ class Property: description: Optional[str] = None # Granularity of the property - # If none, we assume it is a section-level property - granularity: Optional[str] = None + granularity: PropertyGranularity = PropertyGranularity("section") @dataclass @@ -458,6 +461,17 @@ def parse_property_classes(metadata: dict[str, Any]) -> list[PropertyClass]: assert ( "properties" in metadata_cls ), "Metadata class does not contain properties" + properties = [ + Property( + name=p["name"], + type=p["type"], + required=p["required"], + property_values=p["property_values"], + description=p.get("description"), + granularity=PropertyGranularity(p.get("granularity", "section")), + ) + for p in metadata_cls["properties"] + ] classes.append( PropertyClass( name=metadata_cls["name"], @@ -465,10 +479,9 @@ def parse_property_classes(metadata: dict[str, Any]) -> list[PropertyClass]: description=metadata_cls.get("description"), color=metadata_cls.get("color"), sub_types=metadata_cls.get("sub_types"), - properties=[Property(**p) for p in metadata_cls["properties"]], + properties=properties, ) ) - return classes diff --git a/darwin/future/data_objects/properties.py b/darwin/future/data_objects/properties.py index b12ca0a29..27635cf6b 100644 --- a/darwin/future/data_objects/properties.py +++ b/darwin/future/data_objects/properties.py @@ -3,7 +3,8 @@ import json import os from pathlib import Path -from typing import List, Literal, Optional, Tuple +from typing import List, Literal, Optional, Tuple, Union +from enum import Enum from pydantic import field_validator @@ -19,6 +20,12 @@ ] +class PropertyGranularity(str, Enum): + section = "section" + annotation = "annotation" + item = "item" + + class PropertyValue(DefaultDarwin): """ Describes a single option for a property @@ -60,6 +67,8 @@ class FullProperty(DefaultDarwin): type (str): Type of the property required (bool): If the property is required options (List[PropertyOption]): List of all options for the property + granularity (PropertyGranularity): Granularity of the property + """ id: Optional[str] = None @@ -73,6 +82,9 @@ class FullProperty(DefaultDarwin): annotation_class_id: Optional[int] = None property_values: Optional[List[PropertyValue]] = None options: Optional[List[PropertyValue]] = None + granularity: PropertyGranularity = PropertyGranularity("section") + + # model_config = ConfigDict(use_enum_values=True) def to_create_endpoint( self, @@ -87,6 +99,7 @@ def to_create_endpoint( "annotation_class_id": True, "property_values": {"__all__": {"value", "color"}}, "description": True, + "granularity": True, } ) @@ -94,7 +107,8 @@ def to_update_endpoint(self) -> Tuple[str, dict]: if self.id is None: raise ValueError("id must be set") updated_base = self.to_create_endpoint() - del updated_base["annotation_class_id"] # can't update this field + del updated_base["annotation_class_id"] # Can't update this field + del updated_base["granularity"] # Can't update this field return self.id, updated_base @@ -110,6 +124,7 @@ class MetaDataClass(DefaultDarwin): description (Optional[str]): Description of the class color (Optional[str]): Color of the class in the UI sub_types (Optional[List[str]]): Sub types of the class + granularity:(PropertyGranularity): Granularity of the property properties (List[FullProperty]): List of all properties for the class with all options """ @@ -118,6 +133,7 @@ class MetaDataClass(DefaultDarwin): description: Optional[str] = None color: Optional[str] = None sub_types: Optional[List[str]] = None + granularity: PropertyGranularity = PropertyGranularity("section") properties: List[FullProperty] @classmethod @@ -141,13 +157,14 @@ class SelectedProperty(DefaultDarwin): Selected property for an annotation found inside a darwin annotation Attributes: - frame_index (int): Frame index of the annotation + frame_index (int | str): Frame index of the annotation + int for section-level properties, and "global" for annotation-level properties name (str): Name of the property type (str | None): Type of the property (if it exists) value (str): Value of the property """ - frame_index: Optional[int] = None + frame_index: Optional[Union[int, str]] = None name: str type: Optional[str] = None value: Optional[str] = None diff --git a/darwin/future/tests/core/fixtures.py b/darwin/future/tests/core/fixtures.py index 882f0d411..b824232b2 100644 --- a/darwin/future/tests/core/fixtures.py +++ b/darwin/future/tests/core/fixtures.py @@ -8,7 +8,11 @@ from darwin.future.core.client import ClientCore, DarwinConfig from darwin.future.data_objects.dataset import DatasetCore from darwin.future.data_objects.item import ItemCore, ItemLayout, ItemSlot -from darwin.future.data_objects.properties import FullProperty, PropertyValue +from darwin.future.data_objects.properties import ( + FullProperty, + PropertyValue, + PropertyGranularity, +) from darwin.future.data_objects.team import TeamCore, TeamMemberCore from darwin.future.data_objects.team_member_role import TeamMemberRole from darwin.future.data_objects.workflow import WorkflowCore @@ -38,6 +42,7 @@ def base_property_object(base_property_value: PropertyValue) -> FullProperty: annotation_class_id=0, property_values=[base_property_value], options=[base_property_value], + granularity=PropertyGranularity("section"), ) diff --git a/darwin/importer/importer.py b/darwin/importer/importer.py index 138769520..710777750 100644 --- a/darwin/importer/importer.py +++ b/darwin/importer/importer.py @@ -26,6 +26,7 @@ PropertyType, PropertyValue, SelectedProperty, + PropertyGranularity, ) from darwin.item import DatasetItem from darwin.path_utils import is_properties_enabled, parse_metadata @@ -412,6 +413,7 @@ def _import_properties( # if property value is None, update annotation_property_map with empty set if a_prop.value is None: assert t_prop.id is not None + annotation_property_map[annotation_id][str(a_prop.frame_index)][ t_prop.id ] = set() @@ -516,6 +518,7 @@ def _import_properties( slug=client.default_team, annotation_class_id=int(annotation_class_id), property_values=property_values, + granularity=PropertyGranularity(m_prop.granularity.value), ) create_properties.append(full_property) continue @@ -649,6 +652,7 @@ def _import_properties( slug=client.default_team, annotation_class_id=t_prop.annotation_class_id, property_values=extra_property_values, + granularity=PropertyGranularity(t_prop.granularity.value), ) console.print( f"Updating property {full_property.name} ({full_property.type}) with extra metadata values {extra_values}", diff --git a/darwin/utils/utils.py b/darwin/utils/utils.py index 5f57449ad..b7b13c9f3 100644 --- a/darwin/utils/utils.py +++ b/darwin/utils/utils.py @@ -1149,9 +1149,10 @@ def _parse_properties( ) -> Optional[List[SelectedProperty]]: selected_properties = [] for property in properties: + frame_index = property.get("frame_index") selected_properties.append( SelectedProperty( - frame_index=property.get("frame_index", None), + frame_index=frame_index if frame_index is not None else "global", name=property.get("name", None), value=property.get("value", None), ) diff --git a/tests/darwin/client_test.py b/tests/darwin/client_test.py index 1652c2f58..c2f6b4903 100644 --- a/tests/darwin/client_test.py +++ b/tests/darwin/client_test.py @@ -378,6 +378,7 @@ def test_get_team_properties(self, darwin_client: Client) -> None: "slug": "property-question", "team_id": 128, "type": "multi_select", + "granularity": "section", }, ] }, From 943684cfcefce968e02feb74862c0271a3446b41 Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Fri, 13 Sep 2024 15:09:50 +0100 Subject: [PATCH 3/5] Added unit tests for _import_properties() --- ...ta_missing_annotation_property_values.json | 51 +++ ...adata_missing_section_property_values.json | 49 +++ tests/darwin/importer/importer_test.py | 347 ++++++++++++++++++ 3 files changed, 447 insertions(+) create mode 100644 tests/darwin/data/metadata_missing_annotation_property_values.json create mode 100644 tests/darwin/data/metadata_missing_section_property_values.json diff --git a/tests/darwin/data/metadata_missing_annotation_property_values.json b/tests/darwin/data/metadata_missing_annotation_property_values.json new file mode 100644 index 000000000..740f57601 --- /dev/null +++ b/tests/darwin/data/metadata_missing_annotation_property_values.json @@ -0,0 +1,51 @@ +{ + "version": "1.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/metadata/1.0/schema.json", + "classes": [ + { + "name": "test_class", + "type": "bounding_box", + "description": null, + "color": "rgba(255,46,0,1.0)", + "sub_types": [ + "inference" + ], + "properties": [ + { + "name": "existing_property_single_select", + "type": "single_select", + "description": "", + "required": false, + "property_values": [ + { + "value": "1", + "color": "rgba(255,46,0,1.0)" + } + ], + "granularity": "annotation" + }, + { + "name": "existing_property_multi_select", + "type": "multi_select", + "description": "", + "required": false, + "property_values": [ + { + "value": "1", + "color": "rgba(173,255,0,1.0)" + }, + { + "value": "2", + "color": "rgba(255,199,0,1.0)" + } + ], + "granularity": "annotation" + } + ], + "sub_types_settings": { + "inference": {} + } + } + ], + "properties": [] + } \ No newline at end of file diff --git a/tests/darwin/data/metadata_missing_section_property_values.json b/tests/darwin/data/metadata_missing_section_property_values.json new file mode 100644 index 000000000..5adc44c5f --- /dev/null +++ b/tests/darwin/data/metadata_missing_section_property_values.json @@ -0,0 +1,49 @@ +{ + "version": "1.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/metadata/1.0/schema.json", + "classes": [ + { + "name": "test_class", + "type": "bounding_box", + "description": null, + "color": "rgba(255,46,0,1.0)", + "sub_types": [ + "inference" + ], + "properties": [ + { + "name": "existing_property_single_select", + "type": "single_select", + "description": "", + "required": false, + "property_values": [ + { + "value": "1", + "color": "rgba(255,46,0,1.0)" + } + ] + }, + { + "name": "existing_property_multi_select", + "type": "multi_select", + "description": "", + "required": false, + "property_values": [ + { + "value": "1", + "color": "rgba(173,255,0,1.0)" + }, + { + "value": "2", + "color": "rgba(255,199,0,1.0)" + } + ] + } + ], + "sub_types_settings": { + "inference": {} + } + } + ], + "properties": [] + } \ No newline at end of file diff --git a/tests/darwin/importer/importer_test.py b/tests/darwin/importer/importer_test.py index 70cf197df..be717d589 100644 --- a/tests/darwin/importer/importer_test.py +++ b/tests/darwin/importer/importer_test.py @@ -6,6 +6,12 @@ from unittest.mock import MagicMock, Mock, _patch, patch from zipfile import ZipFile +from darwin.future.data_objects.properties import ( + PropertyGranularity, + SelectedProperty, + FullProperty, + PropertyValue, +) import pytest from darwin import datatypes as dt @@ -24,9 +30,49 @@ _parse_empty_masks, _resolve_annotation_classes, _verify_slot_annotation_alignment, + _import_properties, ) +@pytest.fixture +def setup_data(request): + granularity = request.param + client = Mock() + client.default_team = "test_team" + team_slug = "test_team" + annotation_class_ids_map = {("test_class", "polygon"): "123"} + annotations = [ + dt.Annotation( + dt.AnnotationClass("test_class", "polygon"), + {"paths": [[1, 2, 3, 4, 5]]}, + [], + [], + id="annotation_id_1", + properties=[ + SelectedProperty( + frame_index=None if granularity == "annotation" else "0", + name="existing_property_single_select", + type="single_select", + value="1", + ), + SelectedProperty( + frame_index=None if granularity == "annotation" else "0", + name="existing_property_multi_select", + type="multi_select", + value="1", + ), + SelectedProperty( + frame_index=None if granularity == "annotation" else "1", + name="existing_property_multi_select", + type="multi_select", + value="2", + ), + ], + ) + ] + return client, team_slug, annotation_class_ids_map, annotations + + def root_path(x: str) -> str: return f"darwin.importer.importer.{x}" @@ -1465,3 +1511,304 @@ def test_does_not_raise_error_for_darwin_format_with_warnings(): _display_slot_warnings_and_errors(slot_errors, slot_warnings, "darwin", console) assert not slot_errors + + +@patch("darwin.importer.importer._get_team_properties_annotation_lookup") +@pytest.mark.parametrize("setup_data", ["section"], indirect=True) +def test_import_existing_section_level_property_values_without_manifest( + mock_get_team_properties, + setup_data, +): + client, team_slug, annotation_class_ids_map, annotations = setup_data + mock_get_team_properties.return_value = { + ("existing_property_single_select", 123): FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_1"), + ], + ), + ("existing_property_multi_select", 123): FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_2"), + PropertyValue(value="2", id="property_value_id_3"), + ], + ), + } + metadata_path = False + result = _import_properties( + metadata_path, client, annotations, annotation_class_ids_map, team_slug + ) + assert result["annotation_id_1"]["0"]["property_id_1"] == { + "property_value_id_1", + } + assert result["annotation_id_1"]["0"]["property_id_2"] == { + "property_value_id_2", + } + assert result["annotation_id_1"]["1"]["property_id_2"] == { + "property_value_id_3", + } + + +@patch("darwin.importer.importer._get_team_properties_annotation_lookup") +@pytest.mark.parametrize("setup_data", ["section"], indirect=True) +def test_import_new_section_level_property_values_with_manifest( + mock_get_team_properties, + setup_data, +): + client, team_slug, annotation_class_ids_map, annotations = setup_data + mock_get_team_properties.return_value = { + ("existing_property_single_select", 123): FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + property_values=[], + ), + ("existing_property_multi_select", 123): FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_2"), + ], + ), + } + metadata_path = ( + Path(__file__).parents[1] + / "data" + / "metadata_missing_section_property_values.json" + ) + with patch.object(client, "update_property") as mock_update_property: + result = _import_properties( + metadata_path, client, annotations, annotation_class_ids_map, team_slug + ) + assert result["annotation_id_1"]["0"]["property_id_2"] == { + "property_value_id_2", + } + assert mock_update_property.call_args_list[0].kwargs["params"] == FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + description="property-updated-during-annotation-import", + annotation_class_id=123, + slug="test_team", + property_values=[ + PropertyValue(value="1", color="rgba(255,46,0,1.0)"), + ], + ) + assert mock_update_property.call_args_list[1].kwargs["params"] == FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + description="property-updated-during-annotation-import", + annotation_class_id=123, + slug="test_team", + property_values=[ + PropertyValue(value="2", color="rgba(255,199,0,1.0)"), + ], + ) + + +@patch("darwin.importer.importer._get_team_properties_annotation_lookup") +@pytest.mark.parametrize("setup_data", ["section"], indirect=True) +def test_import_new_section_level_properties_with_manifest( + mock_get_team_properties, + setup_data, +): + client, team_slug, annotation_class_ids_map, annotations = setup_data + mock_get_team_properties.return_value = {} + metadata_path = ( + Path(__file__).parents[1] + / "data" + / "metadata_missing_section_property_values.json" + ) + with patch.object(client, "create_property") as mock_create_property: + _import_properties( + metadata_path, client, annotations, annotation_class_ids_map, team_slug + ) + assert mock_create_property.call_args_list[0].kwargs["params"] == FullProperty( + id=None, + position=None, + name="existing_property_single_select", + type="single_select", + required=False, + description="property-created-during-annotation-import", + annotation_class_id=123, + slug="test_team", + team_id=None, + property_values=[ + PropertyValue(value="1", color="rgba(255,46,0,1.0)"), + ], + options=None, + granularity=PropertyGranularity.section, + ) + assert mock_create_property.call_args_list[1].kwargs["params"] == FullProperty( + name="existing_property_multi_select", + type="multi_select", + required=False, + description="property-created-during-annotation-import", + annotation_class_id=123, + slug="test_team", + property_values=[ + PropertyValue(value="1", color="rgba(173,255,0,1.0)"), + PropertyValue(value="2", color="rgba(255,199,0,1.0)"), + ], + ) + + +@patch("darwin.importer.importer._get_team_properties_annotation_lookup") +@pytest.mark.parametrize("setup_data", ["annotation"], indirect=True) +def test_import_existing_annotation_level_property_values_without_manifest( + mock_get_team_properties, + setup_data, +): + client, team_slug, annotation_class_ids_map, annotations = setup_data + mock_get_team_properties.return_value = { + ("existing_property_single_select", 123): FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_1"), + ], + ), + ("existing_property_multi_select", 123): FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_2"), + PropertyValue(value="2", id="property_value_id_3"), + ], + ), + } + metadata_path = False + result = _import_properties( + metadata_path, client, annotations, annotation_class_ids_map, team_slug + ) + assert result["annotation_id_1"]["None"]["property_id_1"] == { + "property_value_id_1", + } + assert result["annotation_id_1"]["None"]["property_id_2"] == { + "property_value_id_2", + "property_value_id_3", + } + + +@patch("darwin.importer.importer._get_team_properties_annotation_lookup") +@pytest.mark.parametrize("setup_data", ["annotation"], indirect=True) +def test_import_new_annotation_level_property_values_with_manifest( + mock_get_team_properties, + setup_data, +): + client, team_slug, annotation_class_ids_map, annotations = setup_data + mock_get_team_properties.return_value = { + ("existing_property_single_select", 123): FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + property_values=[], + ), + ("existing_property_multi_select", 123): FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_2"), + ], + ), + } + metadata_path = ( + Path(__file__).parents[1] + / "data" + / "metadata_missing_annotation_property_values.json" + ) + with patch.object(client, "update_property") as mock_update_property: + result = _import_properties( + metadata_path, client, annotations, annotation_class_ids_map, team_slug + ) + assert result["annotation_id_1"]["None"]["property_id_2"] == { + "property_value_id_2", + } + assert mock_update_property.call_args_list[0].kwargs["params"] == FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + description="property-updated-during-annotation-import", + annotation_class_id=123, + slug="test_team", + property_values=[ + PropertyValue(value="1", color="rgba(255,46,0,1.0)"), + ], + ) + assert mock_update_property.call_args_list[1].kwargs["params"] == FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + description="property-updated-during-annotation-import", + annotation_class_id=123, + slug="test_team", + property_values=[ + PropertyValue(value="2", color="rgba(255,199,0,1.0)"), + ], + ) + + +@patch("darwin.importer.importer._get_team_properties_annotation_lookup") +@pytest.mark.parametrize("setup_data", ["annotation"], indirect=True) +def test_import_new_annotation_level_properties_with_manifest( + mock_get_team_properties, + setup_data, +): + client, team_slug, annotation_class_ids_map, annotations = setup_data + mock_get_team_properties.return_value = {} + metadata_path = ( + Path(__file__).parents[1] + / "data" + / "metadata_missing_annotation_property_values.json" + ) + with patch.object(client, "create_property") as mock_create_property: + _import_properties( + metadata_path, client, annotations, annotation_class_ids_map, team_slug + ) + assert mock_create_property.call_args_list[0].kwargs["params"] == FullProperty( + name="existing_property_single_select", + type="single_select", + required=False, + description="property-created-during-annotation-import", + annotation_class_id=123, + slug="test_team", + property_values=[ + PropertyValue(value="1", color="rgba(255,46,0,1.0)"), + ], + granularity=PropertyGranularity.annotation, + ) + assert mock_create_property.call_args_list[1].kwargs["params"] == FullProperty( + name="existing_property_multi_select", + type="multi_select", + required=False, + description="property-created-during-annotation-import", + annotation_class_id=123, + slug="test_team", + property_values=[ + PropertyValue(value="1", color="rgba(173,255,0,1.0)"), + PropertyValue(value="2", color="rgba(255,199,0,1.0)"), + ], + granularity=PropertyGranularity.annotation, + ) From e325ecb372f62f5834ac0d08a7517e784ec8949e Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Fri, 13 Sep 2024 15:21:50 +0100 Subject: [PATCH 4/5] Fix for albumentations transform --- darwin/future/data_objects/properties.py | 2 -- darwin/torch/transforms.py | 7 +++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/darwin/future/data_objects/properties.py b/darwin/future/data_objects/properties.py index 27635cf6b..f60b917f3 100644 --- a/darwin/future/data_objects/properties.py +++ b/darwin/future/data_objects/properties.py @@ -84,8 +84,6 @@ class FullProperty(DefaultDarwin): options: Optional[List[PropertyValue]] = None granularity: PropertyGranularity = PropertyGranularity("section") - # model_config = ConfigDict(use_enum_values=True) - def to_create_endpoint( self, ) -> dict: diff --git a/darwin/torch/transforms.py b/darwin/torch/transforms.py index fd60cb3c5..e783d4947 100644 --- a/darwin/torch/transforms.py +++ b/darwin/torch/transforms.py @@ -368,8 +368,11 @@ def _pre_process(self, image: np.ndarray, annotation: dict) -> dict: if ( masks is not None and masks.numel() > 0 ): # using numel() to check if tensor is non-empty - print("WE GOT MASKS") - albumentation_dict["masks"] = masks.numpy() + if isinstance(masks, torch.Tensor): + masks = masks.numpy() + if masks.ndim == 3: # Ensure masks is a list of numpy arrays + masks = [masks[i] for i in range(masks.shape[0])] + albumentation_dict["masks"] = masks return albumentation_dict From 84e29000b0c58202c17e22e1273965a5128b6fe5 Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Mon, 16 Sep 2024 11:30:55 +0100 Subject: [PATCH 5/5] Prevent 422s by not performing the same property update/create command multiple times --- darwin/importer/importer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/darwin/importer/importer.py b/darwin/importer/importer.py index 710777750..871160cfe 100644 --- a/darwin/importer/importer.py +++ b/darwin/importer/importer.py @@ -520,7 +520,9 @@ def _import_properties( property_values=property_values, granularity=PropertyGranularity(m_prop.granularity.value), ) - create_properties.append(full_property) + # Don't attempt the same propery creation multiple times + if full_property not in create_properties: + create_properties.append(full_property) continue # check if property value is different in m_prop (.v7/metadata.json) options @@ -568,7 +570,9 @@ def _import_properties( ) ], ) - update_properties.append(full_property) + # Don't attempt the same propery update multiple times + if full_property not in update_properties: + update_properties.append(full_property) continue assert t_prop.id is not None