diff --git a/pyproject.toml b/pyproject.toml index 96d1dcd..c853e59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "sumo-wrapper-python>=1.0.3", "fmu-dataio", "openvds; sys_platform != 'darwin'", + "ert; sys_platform != 'win32'" ] [project.optional-dependencies] diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 8629f1f..948f0c3 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -9,3 +9,4 @@ pyarrow; python_version > "3.6.1" # ecl2df; sys_platform != 'win32' and python_version < "3.11" fmu-dataio OpenVDS; sys_platform != 'darwin' +ert diff --git a/src/fmu/sumo/config_jobs/SUMO_UPLOAD b/src/fmu/sumo/config_jobs/SUMO_UPLOAD index b45b950..b519947 100644 --- a/src/fmu/sumo/config_jobs/SUMO_UPLOAD +++ b/src/fmu/sumo/config_jobs/SUMO_UPLOAD @@ -7,6 +7,9 @@ -- SEARCHPATH: The searchpath relative to the realization root -- e.g "share/results/maps/*.gri" -- SUMO_ENV: The environment to upload to +-- CONFIG_PATH: The path to global_variables.yml + +DEFAULT fmuconfig/output/global_variables.yml STDERR sumo_upload.stderr STDOUT sumo_upload.stdout @@ -14,10 +17,12 @@ STDOUT sumo_upload.stdout EXECUTABLE sumo_upload -ARGLIST +ARGLIST "--config_path" MIN_ARG 2 -MAX_ARG 3 +MAX_ARG 5 ARG_TYPE 0 STRING ARG_TYPE 1 STRING ARG_TYPE 2 STRING +ARG_TYPE 3 STRING +ARG_TYPE 4 STRING \ No newline at end of file diff --git a/src/fmu/sumo/uploader/caseondisk.py b/src/fmu/sumo/uploader/caseondisk.py index 5b66b5e..5efecf6 100644 --- a/src/fmu/sumo/uploader/caseondisk.py +++ b/src/fmu/sumo/uploader/caseondisk.py @@ -81,12 +81,15 @@ def __init__( case_metadata = _load_case_metadata(case_metadata_path) super().__init__(case_metadata, sumo_connection, verbosity) - self._sumo_logger = sumo_connection.api.getLogger("log_2_server_caseondisk") + self._sumo_logger = sumo_connection.api.getLogger( + "log_2_server_caseondisk" + ) self._sumo_logger.setLevel(logging.INFO) # Avoid that logging to sumo-server also is visible in local logging: self._sumo_logger.propagate = False - self._sumo_logger.info("Upload init for sumo_parent_id: " - + str(self._sumo_parent_id)) + self._sumo_logger.info( + "Upload init for sumo_parent_id: " + str(self._sumo_parent_id) + ) def __str__(self): s = f"{self.__class__}, {len(self._files)} files." @@ -139,12 +142,15 @@ def upload_parameters_txt( ): """Upload parameters.txt if it is not present in Sumo for the current realization""" logger.info("Uploading parameters.txt") + print(f"CONFIG_PATH: {glob_var_path}") fmu_id = self.fmu_case_uuid realization_id = self.files[0].metadata["fmu"]["realization"]["uuid"] query = f"fmu.case.uuid:{fmu_id} AND fmu.realization.uuid:{realization_id} AND data.content:parameters" - search_res = self.sumo_connection.api.get("/search", {"$query": query}).json() + search_res = self.sumo_connection.api.get( + "/search", {"$query": query} + ).json() if search_res["hits"]["total"]["value"] == 0: with open(glob_var_path, "r") as variables_yml: diff --git a/src/fmu/sumo/uploader/scripts/sumo_upload.py b/src/fmu/sumo/uploader/scripts/sumo_upload.py index 9d01042..27c9770 100644 --- a/src/fmu/sumo/uploader/scripts/sumo_upload.py +++ b/src/fmu/sumo/uploader/scripts/sumo_upload.py @@ -45,7 +45,7 @@ and where typically refers to // is typically set in the config as it is used also by forward jobs. -It must refer to a valid Sumo environment. Normally this should be set to prod.""" +It must refer to a valid Sumo environment. Normally this should be set to prod.""" def main() -> None: @@ -71,11 +71,17 @@ def main() -> None: env=args.env, metadata_path=args.metadata_path, threads=args.threads, + config_path=args.config_path, ) def sumo_upload_main( - casepath: str, searchpath: str, env: str, metadata_path: str, threads: int + casepath: str, + searchpath: str, + env: str, + metadata_path: str, + threads: int, + config_path: str = "fmuconfig/output/global_variables.yml", ) -> None: """A "main" function that can be used both from command line and from ERT workflow""" @@ -95,7 +101,8 @@ def sumo_upload_main( logger.info("case_metadata_path is %s", case_metadata_path) e = uploader.CaseOnDisk( - case_metadata_path=case_metadata_path, sumo_connection=sumo_connection + case_metadata_path=case_metadata_path, + sumo_connection=sumo_connection, ) # add files to the case on disk object logger.info("Adding files. Search path is %s", searchpath) @@ -110,17 +117,20 @@ def sumo_upload_main( # upload the indexed files logger.info("Starting upload") e.upload(threads=threads, register_case=False) - e.upload_parameters_txt() + e.upload_parameters_txt(glob_var_path=config_path) logger.info("Upload done") except Exception as err: - logger.info( - "Problem related to Sumo upload: " f"{err}" + logger.info("Problem related to Sumo upload: " f"{err}") + warnings.warn("Problem related to Sumo upload: " f"{err}") + _sumo_logger = sumo_connection.api.getLogger( + "log_2_server_sumo_upload" ) - warnings.warn( - "Problem related to Sumo upload: " f"{err}") - _sumo_logger = sumo_connection.api.getLogger("log_2_server_sumo_upload") _sumo_logger.propagate = False - _sumo_logger.warning("Problem related to Sumo upload for case: %s; %s", case_metadata_path, err) + _sumo_logger.warning( + "Problem related to Sumo upload for case: %s; %s", + case_metadata_path, + err, + ) return @@ -144,6 +154,7 @@ def run(self, *args): env=args.env, metadata_path=args.metadata_path, threads=args.threads, + config_path=args.config_path, ) @@ -151,11 +162,20 @@ def _get_parser() -> argparse.ArgumentParser: """Construct parser object for sumo_upload.""" parser = argparse.ArgumentParser() - parser.add_argument("casepath", type=str, help="Absolute path to case root") + parser.add_argument( + "casepath", type=str, help="Absolute path to case root" + ) parser.add_argument( "searchpath", type=str, help="Absolute search path for files to upload" ) parser.add_argument("env", type=str, help="Sumo environment to use.") + parser.add_argument( + "--config_path", + type=str, + help="Absolute path to global variables", + default="fmuconfig/output/global_variables.yml", + ) + parser.add_argument( "--threads", type=int, help="Set number of threads to use.", default=2 ) @@ -165,9 +185,13 @@ def _get_parser() -> argparse.ArgumentParser: help="Case-relative path to case metadata", default="share/metadata/fmu_case.yml", ) - parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output") parser.add_argument( - "--debug", action="store_true", help="Debug output, more verbose than --verbose" + "-v", "--verbose", action="store_true", help="Verbose output" + ) + parser.add_argument( + "--debug", + action="store_true", + help="Debug output, more verbose than --verbose", ) return parser @@ -185,7 +209,9 @@ def _check_arguments(args) -> None: if not Path(args.casepath).is_absolute(): if args.casepath.startswith("<") and args.casepath.endswith(">"): ValueError("ERT variable is not defined: %s", args.casepath) - raise ValueError("Provided casepath must be an absolute path to the case root") + raise ValueError( + "Provided casepath must be an absolute path to the case root" + ) if not Path(args.casepath).exists(): raise ValueError("Provided case path does not exist")