From 1978aaf1914b0a8d2321cef4870d06411069f88c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85dne=20Jacobsen?= Date: Fri, 3 Nov 2023 14:55:56 +0100 Subject: [PATCH 01/11] Add conf_path parameter --- src/fmu/sumo/uploader/caseondisk.py | 13 +++-- src/fmu/sumo/uploader/scripts/sumo_upload.py | 54 +++++++++++++++----- 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/src/fmu/sumo/uploader/caseondisk.py b/src/fmu/sumo/uploader/caseondisk.py index 5b66b5e..b3c1fb4 100644 --- a/src/fmu/sumo/uploader/caseondisk.py +++ b/src/fmu/sumo/uploader/caseondisk.py @@ -81,12 +81,15 @@ def __init__( case_metadata = _load_case_metadata(case_metadata_path) super().__init__(case_metadata, sumo_connection, verbosity) - self._sumo_logger = sumo_connection.api.getLogger("log_2_server_caseondisk") + self._sumo_logger = sumo_connection.api.getLogger( + "log_2_server_caseondisk" + ) self._sumo_logger.setLevel(logging.INFO) # Avoid that logging to sumo-server also is visible in local logging: self._sumo_logger.propagate = False - self._sumo_logger.info("Upload init for sumo_parent_id: " - + str(self._sumo_parent_id)) + self._sumo_logger.info( + "Upload init for sumo_parent_id: " + str(self._sumo_parent_id) + ) def __str__(self): s = f"{self.__class__}, {len(self._files)} files." @@ -144,7 +147,9 @@ def upload_parameters_txt( realization_id = self.files[0].metadata["fmu"]["realization"]["uuid"] query = f"fmu.case.uuid:{fmu_id} AND fmu.realization.uuid:{realization_id} AND data.content:parameters" - search_res = self.sumo_connection.api.get("/search", {"$query": query}).json() + search_res = self.sumo_connection.api.get( + "/search", {"$query": query} + ).json() if search_res["hits"]["total"]["value"] == 0: with open(glob_var_path, "r") as variables_yml: diff --git a/src/fmu/sumo/uploader/scripts/sumo_upload.py b/src/fmu/sumo/uploader/scripts/sumo_upload.py index 9d01042..9336797 100644 --- a/src/fmu/sumo/uploader/scripts/sumo_upload.py +++ b/src/fmu/sumo/uploader/scripts/sumo_upload.py @@ -45,7 +45,7 @@ and where typically refers to // is typically set in the config as it is used also by forward jobs. -It must refer to a valid Sumo environment. Normally this should be set to prod.""" +It must refer to a valid Sumo environment. Normally this should be set to prod.""" def main() -> None: @@ -67,6 +67,7 @@ def main() -> None: sumo_upload_main( casepath=args.casepath, + conf_path=args.conf_path, searchpath=args.searchpath, env=args.env, metadata_path=args.metadata_path, @@ -75,7 +76,12 @@ def main() -> None: def sumo_upload_main( - casepath: str, searchpath: str, env: str, metadata_path: str, threads: int + casepath: str, + conf_path: str, + searchpath: str, + env: str, + metadata_path: str, + threads: int, ) -> None: """A "main" function that can be used both from command line and from ERT workflow""" @@ -95,7 +101,8 @@ def sumo_upload_main( logger.info("case_metadata_path is %s", case_metadata_path) e = uploader.CaseOnDisk( - case_metadata_path=case_metadata_path, sumo_connection=sumo_connection + case_metadata_path=case_metadata_path, + sumo_connection=sumo_connection, ) # add files to the case on disk object logger.info("Adding files. Search path is %s", searchpath) @@ -110,17 +117,20 @@ def sumo_upload_main( # upload the indexed files logger.info("Starting upload") e.upload(threads=threads, register_case=False) - e.upload_parameters_txt() + e.upload_parameters_txt(glob_var_path=conf_path) logger.info("Upload done") except Exception as err: - logger.info( - "Problem related to Sumo upload: " f"{err}" + logger.info("Problem related to Sumo upload: " f"{err}") + warnings.warn("Problem related to Sumo upload: " f"{err}") + _sumo_logger = sumo_connection.api.getLogger( + "log_2_server_sumo_upload" ) - warnings.warn( - "Problem related to Sumo upload: " f"{err}") - _sumo_logger = sumo_connection.api.getLogger("log_2_server_sumo_upload") _sumo_logger.propagate = False - _sumo_logger.warning("Problem related to Sumo upload for case: %s; %s", case_metadata_path, err) + _sumo_logger.warning( + "Problem related to Sumo upload for case: %s; %s", + case_metadata_path, + err, + ) return @@ -140,6 +150,7 @@ def run(self, *args): _check_arguments(args) sumo_upload_main( casepath=args.casepath, + conf_path=args.conf_path, searchpath=args.searchpath, env=args.env, metadata_path=args.metadata_path, @@ -151,11 +162,20 @@ def _get_parser() -> argparse.ArgumentParser: """Construct parser object for sumo_upload.""" parser = argparse.ArgumentParser() - parser.add_argument("casepath", type=str, help="Absolute path to case root") + parser.add_argument( + "casepath", type=str, help="Absolute path to case root" + ) parser.add_argument( "searchpath", type=str, help="Absolute search path for files to upload" ) parser.add_argument("env", type=str, help="Sumo environment to use.") + parser.add_argument( + "conf_path", + type=str, + help="Absolute path to global variables", + default="./fmuconfig/output/global_variables.yml", + ) + parser.add_argument( "--threads", type=int, help="Set number of threads to use.", default=2 ) @@ -165,9 +185,13 @@ def _get_parser() -> argparse.ArgumentParser: help="Case-relative path to case metadata", default="share/metadata/fmu_case.yml", ) - parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output") parser.add_argument( - "--debug", action="store_true", help="Debug output, more verbose than --verbose" + "-v", "--verbose", action="store_true", help="Verbose output" + ) + parser.add_argument( + "--debug", + action="store_true", + help="Debug output, more verbose than --verbose", ) return parser @@ -185,7 +209,9 @@ def _check_arguments(args) -> None: if not Path(args.casepath).is_absolute(): if args.casepath.startswith("<") and args.casepath.endswith(">"): ValueError("ERT variable is not defined: %s", args.casepath) - raise ValueError("Provided casepath must be an absolute path to the case root") + raise ValueError( + "Provided casepath must be an absolute path to the case root" + ) if not Path(args.casepath).exists(): raise ValueError("Provided case path does not exist") From a22d6b1fbd862751ab8d3952395909c11afcb305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85dne=20Jacobsen?= Date: Mon, 6 Nov 2023 16:49:28 +0100 Subject: [PATCH 02/11] cleanup --- src/fmu/sumo/uploader/scripts/sumo_upload.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/fmu/sumo/uploader/scripts/sumo_upload.py b/src/fmu/sumo/uploader/scripts/sumo_upload.py index 9336797..3cb9755 100644 --- a/src/fmu/sumo/uploader/scripts/sumo_upload.py +++ b/src/fmu/sumo/uploader/scripts/sumo_upload.py @@ -67,21 +67,21 @@ def main() -> None: sumo_upload_main( casepath=args.casepath, - conf_path=args.conf_path, searchpath=args.searchpath, env=args.env, metadata_path=args.metadata_path, threads=args.threads, + config_path=args.config_path, ) def sumo_upload_main( casepath: str, - conf_path: str, searchpath: str, env: str, metadata_path: str, threads: int, + config_path: str, ) -> None: """A "main" function that can be used both from command line and from ERT workflow""" @@ -117,7 +117,7 @@ def sumo_upload_main( # upload the indexed files logger.info("Starting upload") e.upload(threads=threads, register_case=False) - e.upload_parameters_txt(glob_var_path=conf_path) + e.upload_parameters_txt(glob_var_path=config_path) logger.info("Upload done") except Exception as err: logger.info("Problem related to Sumo upload: " f"{err}") @@ -150,11 +150,11 @@ def run(self, *args): _check_arguments(args) sumo_upload_main( casepath=args.casepath, - conf_path=args.conf_path, searchpath=args.searchpath, env=args.env, metadata_path=args.metadata_path, threads=args.threads, + config_path=args.config_path, ) @@ -170,7 +170,7 @@ def _get_parser() -> argparse.ArgumentParser: ) parser.add_argument("env", type=str, help="Sumo environment to use.") parser.add_argument( - "conf_path", + "--config_path", type=str, help="Absolute path to global variables", default="./fmuconfig/output/global_variables.yml", From 2164e97214a77e8c1dfba8264288b8c122e8162b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85dne=20Jacobsen?= Date: Tue, 7 Nov 2023 09:13:13 +0100 Subject: [PATCH 03/11] fix default config path --- src/fmu/sumo/uploader/scripts/sumo_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fmu/sumo/uploader/scripts/sumo_upload.py b/src/fmu/sumo/uploader/scripts/sumo_upload.py index 3cb9755..65ef9c0 100644 --- a/src/fmu/sumo/uploader/scripts/sumo_upload.py +++ b/src/fmu/sumo/uploader/scripts/sumo_upload.py @@ -173,7 +173,7 @@ def _get_parser() -> argparse.ArgumentParser: "--config_path", type=str, help="Absolute path to global variables", - default="./fmuconfig/output/global_variables.yml", + default="fmuconfig/output/global_variables.yml", ) parser.add_argument( From e105e0c3fa11a1754670cb3473a4b45593547028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85dne=20Jacobsen?= Date: Wed, 15 Nov 2023 12:16:28 +0100 Subject: [PATCH 04/11] print config pah --- src/fmu/sumo/uploader/caseondisk.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fmu/sumo/uploader/caseondisk.py b/src/fmu/sumo/uploader/caseondisk.py index b3c1fb4..5efecf6 100644 --- a/src/fmu/sumo/uploader/caseondisk.py +++ b/src/fmu/sumo/uploader/caseondisk.py @@ -142,6 +142,7 @@ def upload_parameters_txt( ): """Upload parameters.txt if it is not present in Sumo for the current realization""" logger.info("Uploading parameters.txt") + print(f"CONFIG_PATH: {glob_var_path}") fmu_id = self.fmu_case_uuid realization_id = self.files[0].metadata["fmu"]["realization"]["uuid"] From 1b3fddeda5125917b5046db26e830cff76a8821f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85dne=20Jacobsen?= Date: Thu, 16 Nov 2023 10:03:34 +0100 Subject: [PATCH 05/11] add CONFIG_PATH to arglist --- src/fmu/sumo/config_jobs/SUMO_UPLOAD | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/fmu/sumo/config_jobs/SUMO_UPLOAD b/src/fmu/sumo/config_jobs/SUMO_UPLOAD index b45b950..21a81c5 100644 --- a/src/fmu/sumo/config_jobs/SUMO_UPLOAD +++ b/src/fmu/sumo/config_jobs/SUMO_UPLOAD @@ -7,6 +7,7 @@ -- SEARCHPATH: The searchpath relative to the realization root -- e.g "share/results/maps/*.gri" -- SUMO_ENV: The environment to upload to +-- CONFIG_PATH: The path to global_variables.yml STDERR sumo_upload.stderr STDOUT sumo_upload.stdout @@ -14,10 +15,11 @@ STDOUT sumo_upload.stdout EXECUTABLE sumo_upload -ARGLIST +ARGLIST MIN_ARG 2 MAX_ARG 3 ARG_TYPE 0 STRING ARG_TYPE 1 STRING ARG_TYPE 2 STRING +ARG_TYPE 3 STRING \ No newline at end of file From bf8b02f29d3770c8e425202ffc29367a8787afe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85dne=20Jacobsen?= Date: Thu, 16 Nov 2023 12:43:42 +0100 Subject: [PATCH 06/11] fix config_path arg setup --- src/fmu/sumo/config_jobs/SUMO_UPLOAD | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/fmu/sumo/config_jobs/SUMO_UPLOAD b/src/fmu/sumo/config_jobs/SUMO_UPLOAD index 21a81c5..b519947 100644 --- a/src/fmu/sumo/config_jobs/SUMO_UPLOAD +++ b/src/fmu/sumo/config_jobs/SUMO_UPLOAD @@ -9,17 +9,20 @@ -- SUMO_ENV: The environment to upload to -- CONFIG_PATH: The path to global_variables.yml +DEFAULT fmuconfig/output/global_variables.yml + STDERR sumo_upload.stderr STDOUT sumo_upload.stdout EXECUTABLE sumo_upload -ARGLIST +ARGLIST "--config_path" MIN_ARG 2 -MAX_ARG 3 +MAX_ARG 5 ARG_TYPE 0 STRING ARG_TYPE 1 STRING ARG_TYPE 2 STRING -ARG_TYPE 3 STRING \ No newline at end of file +ARG_TYPE 3 STRING +ARG_TYPE 4 STRING \ No newline at end of file From 87e398ce9b2f5d229e34f60c15615c2048f29a54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85dne=20Jacobsen?= Date: Thu, 16 Nov 2023 14:43:20 +0100 Subject: [PATCH 07/11] add default value to config_path in sumo_upload_main --- src/fmu/sumo/uploader/scripts/sumo_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fmu/sumo/uploader/scripts/sumo_upload.py b/src/fmu/sumo/uploader/scripts/sumo_upload.py index 65ef9c0..27c9770 100644 --- a/src/fmu/sumo/uploader/scripts/sumo_upload.py +++ b/src/fmu/sumo/uploader/scripts/sumo_upload.py @@ -81,7 +81,7 @@ def sumo_upload_main( env: str, metadata_path: str, threads: int, - config_path: str, + config_path: str = "fmuconfig/output/global_variables.yml", ) -> None: """A "main" function that can be used both from command line and from ERT workflow""" From cac28f54417d86f446661ef333af7ce247994eb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85dne=20Jacobsen?= Date: Thu, 16 Nov 2023 14:56:13 +0100 Subject: [PATCH 08/11] add ert dependency --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 9893f36..cd0ba82 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,6 +41,7 @@ install_requires = fmu-dataio urllib3<2.0 openvds; sys_platform != 'darwin' + ert setup_requires = setuptools_scm test_requires = From 14354690dd075a92b4a7ba7fdec295435d209349 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85dne=20Jacobsen?= Date: Thu, 16 Nov 2023 15:34:03 +0100 Subject: [PATCH 09/11] exclude ert dependency on windows --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3725a22..ed2f3d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ "sumo-wrapper-python>=1.0.3", "fmu-dataio", "openvds; sys_platform != 'darwin'", - "ert" + "ert"; sys_platform != "windows" ] [project.optional-dependencies] From b2a11767f6da3b2b9efc1f2d9c150b59852583fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85dne=20Jacobsen?= Date: Thu, 16 Nov 2023 15:36:02 +0100 Subject: [PATCH 10/11] fix dependency exlusion syntax --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ed2f3d6..49783e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ "sumo-wrapper-python>=1.0.3", "fmu-dataio", "openvds; sys_platform != 'darwin'", - "ert"; sys_platform != "windows" + "ert; sys_platform != 'windows'" ] [project.optional-dependencies] From e9a36e541aeeb8c55b1898dc034513c76a93e3a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85dne=20Jacobsen?= Date: Thu, 16 Nov 2023 15:40:52 +0100 Subject: [PATCH 11/11] fix dependency exclude value, windows -> win32 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 49783e6..c853e59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ "sumo-wrapper-python>=1.0.3", "fmu-dataio", "openvds; sys_platform != 'darwin'", - "ert; sys_platform != 'windows'" + "ert; sys_platform != 'win32'" ] [project.optional-dependencies]