Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use case-specific, shared access key for uploading child objects into sumo. #112

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/fmu/sumo/uploader/_sumocase.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,12 @@ def upload(self, threads=4):

logger.debug("files_to_upload: %s", files_to_upload)

sumoclient = self.sumoclient.client_for_case(self._sumo_parent_id)

upload_results = upload_files(
files_to_upload,
self._sumo_parent_id,
self.sumoclient,
sumoclient,
threads,
self.sumo_mode,
self.config_path,
Expand Down
12 changes: 12 additions & 0 deletions src/fmu/sumo/uploader/caseondisk.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import glob
import logging
import os
import time
import warnings
from pathlib import Path

Expand Down Expand Up @@ -156,6 +157,17 @@ def register(self):
sumo_parent_id = self._upload_case_metadata(self.case_metadata)
self._sumo_parent_id = sumo_parent_id

# Give Sumo some time to make the case object searchable.
time.sleep(3)

try:
self.sumoclient.create_shared_access_key_for_case(
self._fmu_case_uuid
)
except Exception as ex:
logger.warn(f"Unable to create shared access key: {ex}")
pass

logger.info("Case registered. SumoID: {}".format(sumo_parent_id))

return sumo_parent_id
Expand Down
80 changes: 27 additions & 53 deletions tests/test_uploader.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,14 @@ def _update_metadata_file_absolute_path(metadata_file):
yaml.dump(parsed_yaml, f)


def _hits_for_case(sumoclient, case_uuid):
query = f"fmu.case.uuid:{case_uuid} AND NOT class:iteration AND NOT class:realization"
search_results = sumoclient.get(
"/search", {"$query": query, "$size": 0}
).json()
return search_results.get("hits").get("total").get("value")


### TESTS ###


Expand Down Expand Up @@ -190,11 +198,7 @@ def test_case_with_restricted_child(token, unique_uuid):
e.upload()
time.sleep(1)

query = f"{e.fmu_case_uuid}"
search_results = sumoclient.get(
"/search", {"$query": query, "$size": 100}
).json()
total = search_results.get("hits").get("total").get("value")
total = _hits_for_case(sumoclient, e.fmu_case_uuid)
assert total == 2

# Delete this case
Expand Down Expand Up @@ -227,11 +231,7 @@ def test_case_with_one_child(token, unique_uuid):
e.upload()
time.sleep(1)

query = f"{e.fmu_case_uuid}"
search_results = sumoclient.get(
"/search", {"$query": query, "$size": 100}
).json()
total = search_results.get("hits").get("total").get("value")
total = _hits_for_case(sumoclient, e.fmu_case_uuid)
assert total == 2

# Delete this case
Expand Down Expand Up @@ -303,13 +303,19 @@ def test_case_with_one_child_and_params(
# sumo_upload_main(case_path, search_string, ENV, search_string, 1)
time.sleep(1)

query = f"{e.fmu_case_uuid}"
query = (
f"{e.fmu_case_uuid} AND NOT class:iteration AND NOT class:realization"
)
search_results = sumoclient.get(
"/search", {"$query": query, "$size": 100}
).json()
hits = search_results["hits"]
results = hits["hits"]
expected_res = ["case", "dictionary", "surface"]
expected_res = [
"case",
"dictionary",
"surface",
]
found_res = []
for result in results:
class_type = result["_source"]["class"]
Expand Down Expand Up @@ -350,11 +356,7 @@ def test_case_with_one_child_with_affiliate_access(token, unique_uuid):
e.upload()
time.sleep(1)

query = f"{e.fmu_case_uuid}"
search_results = sumoclient.get(
"/search", {"$query": query, "$size": 100}
).json()
total = search_results.get("hits").get("total").get("value")
total = _hits_for_case(sumoclient, e.fmu_case_uuid)
assert total == 2

# Delete this case
Expand Down Expand Up @@ -390,11 +392,7 @@ def test_case_with_no_children(token, unique_uuid):
warnings_record[0].message.args[0].startswith("No files found")
)

query = f"{e.fmu_case_uuid}"
search_results = sumoclient.get(
"/search", {"$query": query, "$size": 100}
).json()
total = search_results.get("hits").get("total").get("value")
total = _hits_for_case(sumoclient, e.fmu_case_uuid)
assert total == 1

# Delete this case
Expand Down Expand Up @@ -442,11 +440,7 @@ def test_missing_child_metadata(token, unique_uuid):
time.sleep(1)

# Assert parent and valid child is on Sumo
query = f"{e.fmu_case_uuid}"
search_results = sumoclient.get(
"/search", {"$query": query, "$size": 100}
).json()
total = search_results.get("hits").get("total").get("value")
total = _hits_for_case(sumoclient, e.fmu_case_uuid)
assert total == 2

# Delete this case
Expand Down Expand Up @@ -512,11 +506,7 @@ def test_invalid_yml_in_child_metadata(token, unique_uuid):
time.sleep(1)

# Assert parent and only 1 valid child are on Sumo
query = f"{e.fmu_case_uuid}"
search_results = sumoclient.get(
"/search", {"$query": query, "$size": 100}
).json()
total = search_results.get("hits").get("total").get("value")
total = _hits_for_case(sumoclient, e.fmu_case_uuid)
assert total == 2

# Delete this case
Expand Down Expand Up @@ -576,11 +566,7 @@ def test_schema_error_in_child(token, unique_uuid):
time.sleep(1)

# Assert parent and valid child are on Sumo
query = f"{e.fmu_case_uuid}"
search_results = sumoclient.get(
"/search", {"$query": query, "$size": 100}
).json()
total = search_results.get("hits").get("total").get("value")
total = _hits_for_case(sumoclient, e.fmu_case_uuid)
assert total == 2

# Delete this case
Expand Down Expand Up @@ -656,7 +642,7 @@ def test_seismic_openvds_file(token, unique_uuid):
time.sleep(1)

# Read the parent object from Sumo
query = f"_sumo.parent_object:{e.fmu_case_uuid}"
query = f"_sumo.parent_object:{e.fmu_case_uuid} AND NOT class:iteration AND NOT class:realization"
search_results = sumoclient.get(
"/search", {"$query": query, "$size": 100}
).json()
Expand Down Expand Up @@ -787,11 +773,7 @@ def test_sumo_mode_default(token, unique_uuid):
time.sleep(1)

# Assert parent and valid child are on Sumo
query = f"{e.fmu_case_uuid}"
search_results = sumoclient.get(
"/search", {"$query": query, "$size": 100}
).json()
total = search_results.get("hits").get("total").get("value")
total = _hits_for_case(sumoclient, e.fmu_case_uuid)
assert total == 2

# Assert that child file and metadatafile are not deleted
Expand Down Expand Up @@ -839,11 +821,7 @@ def test_sumo_mode_copy(token, unique_uuid):
time.sleep(1)

# Assert parent and valid child are on Sumo
query = f"{e.fmu_case_uuid}"
search_results = sumoclient.get(
"/search", {"$query": query, "$size": 100}
).json()
total = search_results.get("hits").get("total").get("value")
total = _hits_for_case(sumoclient, e.fmu_case_uuid)
assert total == 2

# Assert that child file and metadatafile are not deleted
Expand Down Expand Up @@ -902,11 +880,7 @@ def test_sumo_mode_move(token, unique_uuid):
time.sleep(1)

# Assert parent and valid child are on Sumo
query = f"{e.fmu_case_uuid}"
search_results = sumoclient.get(
"/search", {"$query": query, "$size": 100}
).json()
total = search_results.get("hits").get("total").get("value")
total = _hits_for_case(sumoclient, e.fmu_case_uuid)
assert total == 2

# Assert that child file and metadatafile are deleted
Expand Down
Loading