From d08b6a3436ee9a6905942b4395f8b8f65174393f Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Wed, 23 Oct 2024 22:34:29 +0100 Subject: [PATCH] Import item-level properties without annotations --- darwin/importer/importer.py | 3 +- e2e_tests/cli/test_import.py | 11 ++++ .../image_1.json | 52 +++++++++++++++++++ .../image_2.json | 52 +++++++++++++++++++ .../image_3.json | 52 +++++++++++++++++++ .../image_4.json | 52 +++++++++++++++++++ .../image_5.json | 52 +++++++++++++++++++ .../image_6.json | 52 +++++++++++++++++++ .../image_7.json | 52 +++++++++++++++++++ .../image_8.json | 52 +++++++++++++++++++ 10 files changed, 429 insertions(+), 1 deletion(-) create mode 100644 e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_1.json create mode 100644 e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_2.json create mode 100644 e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_3.json create mode 100644 e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_4.json create mode 100644 e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_5.json create mode 100644 e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_6.json create mode 100644 e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_7.json create mode 100644 e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_8.json diff --git a/darwin/importer/importer.py b/darwin/importer/importer.py index 69b015e63..72780a9d4 100644 --- a/darwin/importer/importer.py +++ b/darwin/importer/importer.py @@ -1419,7 +1419,8 @@ def process_local_file(local_file): files_to_not_track = [ file_to_track for file_to_track in parsed_files - if not file_to_track.annotations and (not delete_for_empty) + if not (file_to_track.annotations or file_to_track.item_properties) + and (not delete_for_empty) ] for file in files_to_not_track: diff --git a/e2e_tests/cli/test_import.py b/e2e_tests/cli/test_import.py index afd6fa082..0480eb714 100644 --- a/e2e_tests/cli/test_import.py +++ b/e2e_tests/cli/test_import.py @@ -319,6 +319,17 @@ def test_import_existing_item_level_properties( ) +def test_item_level_properties_can_be_imported_without_annotations( + local_dataset: E2EDataset, config_values: ConfigValues +) -> None: + run_import_test( + local_dataset, + config_values, + item_type="single_slotted", + annotations_subdir="image_annotations_item_level_properties_no_annotations", + ) + + def test_item_level_property_classes_are_created_on_import( local_dataset: E2EDataset, config_values: ConfigValues ) -> None: diff --git a/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_1.json b/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_1.json new file mode 100644 index 000000000..4428eb8b7 --- /dev/null +++ b/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_1.json @@ -0,0 +1,52 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_1", + "path": "/", + "source_info": { + "item_id": "01920b92-1d5d-94a4-6fbe-8a4d7f9fa15d", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-94a4-6fbe-8a4d7f9fa15d" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/2ec69e41-91b2-4155-9b05-6ed995677b1e/thumbnail", + "source_files": [ + { + "file_name": "image_1", + "storage_key": "darwin-py/images/image_1.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/9dfc5eac-bf16-4380-a148-9fff6e63b9f0" + } + ] + } + ] + }, + "annotations": [], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_2.json b/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_2.json new file mode 100644 index 000000000..217492d92 --- /dev/null +++ b/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_2.json @@ -0,0 +1,52 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_2", + "path": "/", + "source_info": { + "item_id": "01920b92-1d5d-ea77-8fa4-16378bafedb3", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-ea77-8fa4-16378bafedb3" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/5e0b3d9d-9bf8-4166-8949-6ab7392161ad/thumbnail", + "source_files": [ + { + "file_name": "image_2", + "storage_key": "darwin-py/images/image_2.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/4920b12a-1706-47f1-b084-2d2234ed1151" + } + ] + } + ] + }, + "annotations": [], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_3.json b/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_3.json new file mode 100644 index 000000000..a0d4f9af9 --- /dev/null +++ b/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_3.json @@ -0,0 +1,52 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_3", + "path": "/dir1", + "source_info": { + "item_id": "01920b92-1d5d-e8ad-986f-ad4942f1bbfc", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-e8ad-986f-ad4942f1bbfc" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/ddd13905-9bbb-4fab-9642-bf4604686fda/thumbnail", + "source_files": [ + { + "file_name": "image_3", + "storage_key": "darwin-py/images/image_3.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/30ec0f13-caaa-4374-be5a-e90b3493fb73" + } + ] + } + ] + }, + "annotations": [], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_4.json b/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_4.json new file mode 100644 index 000000000..7da556fb6 --- /dev/null +++ b/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_4.json @@ -0,0 +1,52 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_4", + "path": "/dir1", + "source_info": { + "item_id": "01920b92-1d5d-8b50-17e9-c0f178e6eee6", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-8b50-17e9-c0f178e6eee6" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/3c731d84-7d7f-4ac8-bbd9-0d53f1d47195/thumbnail", + "source_files": [ + { + "file_name": "image_4", + "storage_key": "darwin-py/images/image_4.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/609ba1a4-79da-4743-b331-e57ccd9ee518" + } + ] + } + ] + }, + "annotations": [], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_5.json b/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_5.json new file mode 100644 index 000000000..114a849eb --- /dev/null +++ b/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_5.json @@ -0,0 +1,52 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_5", + "path": "/dir2", + "source_info": { + "item_id": "01920b92-1d5d-55bf-d705-8b39dea7fde6", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-55bf-d705-8b39dea7fde6" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/8f95e81c-def7-4973-9152-6d0fc39e1473/thumbnail", + "source_files": [ + { + "file_name": "image_5", + "storage_key": "darwin-py/images/image_5.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/08448a07-4e23-41f9-abbd-0dc149ef2be4" + } + ] + } + ] + }, + "annotations": [], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_6.json b/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_6.json new file mode 100644 index 000000000..7f0a0be13 --- /dev/null +++ b/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_6.json @@ -0,0 +1,52 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_6", + "path": "/dir2", + "source_info": { + "item_id": "01920b92-1d5d-1832-3a09-1f38557c57b4", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-1832-3a09-1f38557c57b4" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/4950b608-00a1-4e73-b746-bfe1ea0a1ab6/thumbnail", + "source_files": [ + { + "file_name": "image_6", + "storage_key": "darwin-py/images/image_6.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/9e070e8c-03b3-40b7-a3cb-6da6bcc8d4ed" + } + ] + } + ] + }, + "annotations": [], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_7.json b/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_7.json new file mode 100644 index 000000000..82f5e7861 --- /dev/null +++ b/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_7.json @@ -0,0 +1,52 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_7", + "path": "/dir1/dir3", + "source_info": { + "item_id": "01920b92-1d5d-46ee-5117-53ba0d29d1b0", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-46ee-5117-53ba0d29d1b0" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/1e2f63eb-b7fc-482f-91f3-8caa242e63cb/thumbnail", + "source_files": [ + { + "file_name": "image_7", + "storage_key": "darwin-py/images/image_7.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/20de7c08-20dc-4f16-b559-bbcce2f7b319" + } + ] + } + ] + }, + "annotations": [], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_8.json b/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_8.json new file mode 100644 index 000000000..72a9e2e40 --- /dev/null +++ b/e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_8.json @@ -0,0 +1,52 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_8", + "path": "/dir1/dir3", + "source_info": { + "item_id": "01920b92-1d5e-908e-7b24-3d339ea72237", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5e-908e-7b24-3d339ea72237" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/ace6c9a2-d39a-43df-9fd2-9f124176810a/thumbnail", + "source_files": [ + { + "file_name": "image_8", + "storage_key": "darwin-py/images/image_8.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/141cdb56-2494-4052-bce2-b22673e6ad68" + } + ] + } + ] + }, + "annotations": [], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file