diff --git a/poetry.lock b/poetry.lock index 64271a728..719f69c94 100644 --- a/poetry.lock +++ b/poetry.lock @@ -865,13 +865,13 @@ test-randomorder = ["pytest-randomly"] [[package]] name = "dcicutils" -version = "8.7.1.1b3" +version = "8.7.1.1b4" description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" optional = false python-versions = ">=3.8,<3.12" files = [ - {file = "dcicutils-8.7.1.1b3-py3-none-any.whl", hash = "sha256:d0abae384c9a68fff268a7d7c45911d1ffd8ca2ae7a5ca51d8b50075ccfc40a7"}, - {file = "dcicutils-8.7.1.1b3.tar.gz", hash = "sha256:d6ce6bafd47b2833b716064e94b7954e6b80425ba8777b4fa7b4e6c403f16cee"}, + {file = "dcicutils-8.7.1.1b4-py3-none-any.whl", hash = "sha256:907e51bd3969adf24a778964904474ece4850b45932b9924ed0a98d5f3d11b42"}, + {file = "dcicutils-8.7.1.1b4.tar.gz", hash = "sha256:49d5234e811d295977e51075718bab3fd82f1fac180be074e453ba01ec331f9c"}, ] [package.dependencies] @@ -3275,4 +3275,4 @@ test = ["zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.12" -content-hash = "d8be07912abc1e18f75e7b4353d91408d2db405f19a6dfe5038c113865841b5b" +content-hash = "b96567e1df0d4a350a30e38c89dd90072957971fb7ab0ba5f2c1d143a9dad206" diff --git a/pyproject.toml b/pyproject.toml index 84118b73e..fe535b524 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicsnovault" -version = "11.9.0.1b3" # TODO: To become 11.10.0 +version = "11.9.0.1b4" # TODO: To become 11.10.0 description = "Storage support for 4DN Data Portals." authors = ["4DN-DCIC Team "] license = "MIT" @@ -43,7 +43,7 @@ boto3 = ">=1.28.62" # no particular version required, but this speeds up search botocore = ">=1.31.62" # no particular version required, but this speeds up search elasticsearch = "7.13.4" # versions >= 7.14.0 lock out AWS ES elasticsearch_dsl = "^7.4.0" -dcicutils = "^8.7.1.1b3" +dcicutils = "^8.7.1.1b4" future = ">=0.15.2,<1" html5lib = ">=1.1" # experimental, should be OK now that we're not using moto server humanfriendly = "^1.44.9" diff --git a/snovault/loadxl.py b/snovault/loadxl.py index b3f2fd09d..6380fdccc 100644 --- a/snovault/loadxl.py +++ b/snovault/loadxl.py @@ -11,7 +11,7 @@ import re import structlog import traceback -from typing import List, Optional, Union +from typing import List, Optional, Tuple, Union from webtest import TestApp from webtest.response import TestResponse as TestAppResponse import uuid @@ -19,6 +19,7 @@ from pyramid.response import Response from pyramid.router import Router from pyramid.view import view_config +from dcicutils.data_readers import RowReader from dcicutils.misc_utils import ignored, environ_bool, to_camel_case, VirtualApp from dcicutils.secrets_utils import assume_identity from snovault.util import debug_log @@ -389,6 +390,17 @@ def get_response_uuid(response: TestAppResponse) -> Optional[str]: return response.json.get("uuid") or response.json.get("@graph", [{}])[0].get("uuid") +def normalize_deleted_properties(data: dict) -> Tuple[dict, List[str]]: + normalized_data = {} + deleted_properties = [] + for property_name, property_value in data.items(): + if property_value == RowReader.CELL_DELETION_SENTINEL: + deleted_properties.append(property_name) + else: + normalized_data[property_name] = property_value + return normalized_data, deleted_properties + + def load_all_gen(testapp, inserts, docsdir, overwrite=True, itype=None, from_json=False, patch_only=False, post_only=False, skip_types=None, validate_only=False, continue_on_exception: bool = False, verbose=False): @@ -572,6 +584,7 @@ def get_schema_info(type_name: str) -> (list, list): filename = "" yield str.encode(f'SKIP: {identifying_value}{" " + a_type if verbose else ""}{filename}\n') else: + an_item, _ = normalize_deleted_properties(an_item) if post_only: to_post = an_item else: @@ -643,7 +656,10 @@ def get_schema_info(type_name: str) -> (list, list): identifying_path = get_identifying_path(an_item, a_type, identifying_properties) if not identifying_path: raise Exception("Item has no uuid nor any other identifying property; cannot PATCH.") - res = testapp.patch_json(identifying_path, an_item) + normalized_item, deleted_properties = normalize_deleted_properties(an_item) + if deleted_properties: + identifying_path += f"?delete_fields={','.join(deleted_properties)}" + res = testapp.patch_json(identifying_path, normalized_item) assert res.status_code == 200 patched += 1 # yield bytes to work with Response.app_iter