From ee6c3b93776e37d02b3334c2281ec2b86d191723 Mon Sep 17 00:00:00 2001 From: Raymond Wiker Date: Thu, 19 Dec 2024 13:32:40 +0100 Subject: [PATCH 1/6] Use case-specific, shared access key for uploading child objects into sumo. --- src/fmu/sumo/uploader/_sumocase.py | 4 +++- src/fmu/sumo/uploader/caseondisk.py | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/fmu/sumo/uploader/_sumocase.py b/src/fmu/sumo/uploader/_sumocase.py index 3334988..0cdc1d3 100644 --- a/src/fmu/sumo/uploader/_sumocase.py +++ b/src/fmu/sumo/uploader/_sumocase.py @@ -86,10 +86,12 @@ def upload(self, threads=4): logger.debug("files_to_upload: %s", files_to_upload) + sumoclient = self.sumoclient.client_for_case(self._sumo_parent_id) + upload_results = upload_files( files_to_upload, self._sumo_parent_id, - self.sumoclient, + sumoclient, threads, self.sumo_mode, self.config_path, diff --git a/src/fmu/sumo/uploader/caseondisk.py b/src/fmu/sumo/uploader/caseondisk.py index ccdc336..b1a3a22 100644 --- a/src/fmu/sumo/uploader/caseondisk.py +++ b/src/fmu/sumo/uploader/caseondisk.py @@ -156,6 +156,14 @@ def register(self): sumo_parent_id = self._upload_case_metadata(self.case_metadata) self._sumo_parent_id = sumo_parent_id + try: + self.sumoclient.create_shared_access_key_for_case( + self._fmu_case_uuid + ) + except Exception as ex: + logger.warn(f"Unable to create shared access key: {ex}") + pass + logger.info("Case registered. SumoID: {}".format(sumo_parent_id)) return sumo_parent_id From 58bbf900fa630ba724764771d5bf9ee830ea3d2a Mon Sep 17 00:00:00 2001 From: Raymond Wiker Date: Fri, 20 Dec 2024 10:05:33 +0100 Subject: [PATCH 2/6] fix: give sumo some time to make case object searchable. --- src/fmu/sumo/uploader/caseondisk.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/fmu/sumo/uploader/caseondisk.py b/src/fmu/sumo/uploader/caseondisk.py index b1a3a22..5cbb166 100644 --- a/src/fmu/sumo/uploader/caseondisk.py +++ b/src/fmu/sumo/uploader/caseondisk.py @@ -3,6 +3,7 @@ import glob import logging import os +import time import warnings from pathlib import Path @@ -156,6 +157,9 @@ def register(self): sumo_parent_id = self._upload_case_metadata(self.case_metadata) self._sumo_parent_id = sumo_parent_id + # Give Sumo some time to make the case object searchable. + time.sleep(3) + try: self.sumoclient.create_shared_access_key_for_case( self._fmu_case_uuid From 9e2569d0a4b0196dd12750b393f4682d62a8d7ac Mon Sep 17 00:00:00 2001 From: Raymond Wiker Date: Fri, 20 Dec 2024 10:06:21 +0100 Subject: [PATCH 3/6] fix: refactor, and update tests to allow for iteration and realization objects. --- tests/test_uploader.py | 68 +++++++++++------------------------------- 1 file changed, 18 insertions(+), 50 deletions(-) diff --git a/tests/test_uploader.py b/tests/test_uploader.py index 820ce07..17c2e35 100644 --- a/tests/test_uploader.py +++ b/tests/test_uploader.py @@ -73,6 +73,14 @@ def _update_metadata_file_absolute_path(metadata_file): yaml.dump(parsed_yaml, f) +def _hits_for_case(sumoclient, case_uuid): + query = f"fmu.case.uuid:{case_uuid} AND NOT class:iteration AND NOT class:realization" + search_results = sumoclient.get( + "/search", {"$query": query, "$size": 0} + ).json() + return search_results.get("hits").get("total").get("value") + + ### TESTS ### @@ -190,11 +198,7 @@ def test_case_with_restricted_child(token, unique_uuid): e.upload() time.sleep(1) - query = f"{e.fmu_case_uuid}" - search_results = sumoclient.get( - "/search", {"$query": query, "$size": 100} - ).json() - total = search_results.get("hits").get("total").get("value") + total = _hits_for_case(sumoclient, e.fmu_case_uuid) assert total == 2 # Delete this case @@ -227,11 +231,7 @@ def test_case_with_one_child(token, unique_uuid): e.upload() time.sleep(1) - query = f"{e.fmu_case_uuid}" - search_results = sumoclient.get( - "/search", {"$query": query, "$size": 100} - ).json() - total = search_results.get("hits").get("total").get("value") + total = _hits_for_case(sumoclient, e.fmu_case_uuid) assert total == 2 # Delete this case @@ -350,11 +350,7 @@ def test_case_with_one_child_with_affiliate_access(token, unique_uuid): e.upload() time.sleep(1) - query = f"{e.fmu_case_uuid}" - search_results = sumoclient.get( - "/search", {"$query": query, "$size": 100} - ).json() - total = search_results.get("hits").get("total").get("value") + total = _hits_for_case(sumoclient, e.fmu_case_uuid) assert total == 2 # Delete this case @@ -390,11 +386,7 @@ def test_case_with_no_children(token, unique_uuid): warnings_record[0].message.args[0].startswith("No files found") ) - query = f"{e.fmu_case_uuid}" - search_results = sumoclient.get( - "/search", {"$query": query, "$size": 100} - ).json() - total = search_results.get("hits").get("total").get("value") + total = _hits_for_case(sumoclient, e.fmu_case_uuid) assert total == 1 # Delete this case @@ -442,11 +434,7 @@ def test_missing_child_metadata(token, unique_uuid): time.sleep(1) # Assert parent and valid child is on Sumo - query = f"{e.fmu_case_uuid}" - search_results = sumoclient.get( - "/search", {"$query": query, "$size": 100} - ).json() - total = search_results.get("hits").get("total").get("value") + total = _hits_for_case(sumoclient, e.fmu_case_uuid) assert total == 2 # Delete this case @@ -512,11 +500,7 @@ def test_invalid_yml_in_child_metadata(token, unique_uuid): time.sleep(1) # Assert parent and only 1 valid child are on Sumo - query = f"{e.fmu_case_uuid}" - search_results = sumoclient.get( - "/search", {"$query": query, "$size": 100} - ).json() - total = search_results.get("hits").get("total").get("value") + total = _hits_for_case(sumoclient, e.fmu_case_uuid) assert total == 2 # Delete this case @@ -576,11 +560,7 @@ def test_schema_error_in_child(token, unique_uuid): time.sleep(1) # Assert parent and valid child are on Sumo - query = f"{e.fmu_case_uuid}" - search_results = sumoclient.get( - "/search", {"$query": query, "$size": 100} - ).json() - total = search_results.get("hits").get("total").get("value") + total = _hits_for_case(sumoclient, e.fmu_case_uuid) assert total == 2 # Delete this case @@ -787,11 +767,7 @@ def test_sumo_mode_default(token, unique_uuid): time.sleep(1) # Assert parent and valid child are on Sumo - query = f"{e.fmu_case_uuid}" - search_results = sumoclient.get( - "/search", {"$query": query, "$size": 100} - ).json() - total = search_results.get("hits").get("total").get("value") + total = _hits_for_case(sumoclient, e.fmu_case_uuid) assert total == 2 # Assert that child file and metadatafile are not deleted @@ -839,11 +815,7 @@ def test_sumo_mode_copy(token, unique_uuid): time.sleep(1) # Assert parent and valid child are on Sumo - query = f"{e.fmu_case_uuid}" - search_results = sumoclient.get( - "/search", {"$query": query, "$size": 100} - ).json() - total = search_results.get("hits").get("total").get("value") + total = _hits_for_case(sumoclient, e.fmu_case_uuid) assert total == 2 # Assert that child file and metadatafile are not deleted @@ -902,11 +874,7 @@ def test_sumo_mode_move(token, unique_uuid): time.sleep(1) # Assert parent and valid child are on Sumo - query = f"{e.fmu_case_uuid}" - search_results = sumoclient.get( - "/search", {"$query": query, "$size": 100} - ).json() - total = search_results.get("hits").get("total").get("value") + total = _hits_for_case(sumoclient, e.fmu_case_uuid) assert total == 2 # Assert that child file and metadatafile are deleted From 26898e8701896eb4855c0dbc8dbe5d9a972cb64b Mon Sep 17 00:00:00 2001 From: Raymond Wiker Date: Fri, 20 Dec 2024 11:31:56 +0100 Subject: [PATCH 4/6] Adjust expected results to also allow iteration and realization objects. --- tests/test_uploader.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_uploader.py b/tests/test_uploader.py index 17c2e35..3ba7d58 100644 --- a/tests/test_uploader.py +++ b/tests/test_uploader.py @@ -309,7 +309,13 @@ def test_case_with_one_child_and_params( ).json() hits = search_results["hits"] results = hits["hits"] - expected_res = ["case", "dictionary", "surface"] + expected_res = [ + "case", + "dictionary", + "surface", + "iteration", + "realization", + ] found_res = [] for result in results: class_type = result["_source"]["class"] From e03d894bab6c9170dbe62291f33def7cf2606a69 Mon Sep 17 00:00:00 2001 From: Raymond Wiker Date: Fri, 20 Dec 2024 11:42:54 +0100 Subject: [PATCH 5/6] Adjust tests. --- tests/test_uploader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_uploader.py b/tests/test_uploader.py index 3ba7d58..75813d5 100644 --- a/tests/test_uploader.py +++ b/tests/test_uploader.py @@ -303,7 +303,9 @@ def test_case_with_one_child_and_params( # sumo_upload_main(case_path, search_string, ENV, search_string, 1) time.sleep(1) - query = f"{e.fmu_case_uuid}" + query = ( + f"{e.fmu_case_uuid} AND NOT class:iteration AND NOT class:realization" + ) search_results = sumoclient.get( "/search", {"$query": query, "$size": 100} ).json() @@ -313,8 +315,6 @@ def test_case_with_one_child_and_params( "case", "dictionary", "surface", - "iteration", - "realization", ] found_res = [] for result in results: From 50945c52abfc58b0d57318e086a52aba38601acd Mon Sep 17 00:00:00 2001 From: Raymond Wiker Date: Fri, 20 Dec 2024 12:02:09 +0100 Subject: [PATCH 6/6] Adjust tests, again. --- tests/test_uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_uploader.py b/tests/test_uploader.py index 75813d5..9cb9eeb 100644 --- a/tests/test_uploader.py +++ b/tests/test_uploader.py @@ -642,7 +642,7 @@ def test_seismic_openvds_file(token, unique_uuid): time.sleep(1) # Read the parent object from Sumo - query = f"_sumo.parent_object:{e.fmu_case_uuid}" + query = f"_sumo.parent_object:{e.fmu_case_uuid} AND NOT class:iteration AND NOT class:realization" search_results = sumoclient.get( "/search", {"$query": query, "$size": 100} ).json()