From 21e2aece449bcf8c5499190ea21e96bba4087ddf Mon Sep 17 00:00:00 2001 From: Roy Willy Haug <5484176+roywilly@users.noreply.github.com> Date: Mon, 15 Apr 2024 13:52:48 +0200 Subject: [PATCH] Log to stdout. Minimal logging for create case and workflow job, ordinary logging in forward job. (#54) * Log to stdout. Minimal logging for create case and workflow job * Remove unused var --- src/fmu/sumo/uploader/_fileondisk.py | 10 +++--- src/fmu/sumo/uploader/_fileonjob.py | 5 +-- src/fmu/sumo/uploader/_logger.py | 36 ++++++++++++++++++++ src/fmu/sumo/uploader/_sumocase.py | 7 ++-- src/fmu/sumo/uploader/_sumofile.py | 5 +-- src/fmu/sumo/uploader/caseondisk.py | 7 ++-- src/fmu/sumo/uploader/caseonjob.py | 5 +-- src/fmu/sumo/uploader/scripts/sumo_upload.py | 15 ++++++-- 8 files changed, 71 insertions(+), 19 deletions(-) create mode 100644 src/fmu/sumo/uploader/_logger.py diff --git a/src/fmu/sumo/uploader/_fileondisk.py b/src/fmu/sumo/uploader/_fileondisk.py index 630934d..6b118fc 100644 --- a/src/fmu/sumo/uploader/_fileondisk.py +++ b/src/fmu/sumo/uploader/_fileondisk.py @@ -15,11 +15,12 @@ import yaml from fmu.sumo.uploader._sumofile import SumoFile +from fmu.sumo.uploader._logger import get_uploader_logger + # pylint: disable=C0103 # allow non-snake case variable names -logger = logging.getLogger(__name__) -logger.setLevel(logging.CRITICAL) +logger = get_uploader_logger() def path_to_yaml_path(path): @@ -54,15 +55,14 @@ def file_to_byte_string(path): class FileOnDisk(SumoFile): - def __init__(self, path: str, metadata_path=None, verbosity="INFO"): + def __init__(self, path: str, metadata_path=None, verbosity="WARNING"): """ path (str): Path to file metadata_path (str): Path to metadata file. If not provided, path will be derived from file path. """ - self.verbosity = verbosity - logger.setLevel(level=self.verbosity) + logger.setLevel(level=verbosity) self.metadata_path = ( metadata_path if metadata_path else path_to_yaml_path(path) diff --git a/src/fmu/sumo/uploader/_fileonjob.py b/src/fmu/sumo/uploader/_fileonjob.py index 8fed27a..d5bc438 100644 --- a/src/fmu/sumo/uploader/_fileonjob.py +++ b/src/fmu/sumo/uploader/_fileonjob.py @@ -11,11 +11,12 @@ import base64 from fmu.sumo.uploader._sumofile import SumoFile +from fmu.sumo.uploader._logger import get_uploader_logger + # pylint: disable=C0103 # allow non-snake case variable names -logger = logging.getLogger(__name__) -logger.setLevel(logging.CRITICAL) +logger = get_uploader_logger() class FileOnJob(SumoFile): def __init__(self, byte_string: str, metadata): diff --git a/src/fmu/sumo/uploader/_logger.py b/src/fmu/sumo/uploader/_logger.py new file mode 100644 index 0000000..06bcb3f --- /dev/null +++ b/src/fmu/sumo/uploader/_logger.py @@ -0,0 +1,36 @@ +import logging +import sys +import os +import time + +# def getLogger(module_name="subscript"): +def get_uploader_logger(): + # pylint: disable=invalid-name + """Provides a unified logger for fmu-sumo-uploader. + + Code is copied from + https://github.com/equinor/subscript/blob/main/src/subscript/__init__.py + + Logging output is split by logging levels (split between WARNING and ERROR) + to stdout and stderr, each log occurs in only one of the streams. + + Returns: + A logger object + """ + logger = logging.getLogger("fmu.sumo.uploader") + logger.propagate = False # Avoids duplicate logging + + if not len(logger.handlers): + formatter = logging.Formatter("%(asctime)s:%(levelname)s:%(name)s:%(message)s") + + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.addFilter(lambda record: record.levelno < logging.ERROR) + stdout_handler.setFormatter(formatter) + logger.addHandler(stdout_handler) + + stderr_handler = logging.StreamHandler(sys.stderr) + stderr_handler.addFilter(lambda record: record.levelno >= logging.ERROR) + stderr_handler.setFormatter(formatter) + logger.addHandler(stderr_handler) + + return logger diff --git a/src/fmu/sumo/uploader/_sumocase.py b/src/fmu/sumo/uploader/_sumocase.py index ea6d9fa..17909a1 100644 --- a/src/fmu/sumo/uploader/_sumocase.py +++ b/src/fmu/sumo/uploader/_sumocase.py @@ -13,16 +13,17 @@ from fmu.sumo.uploader._upload_files import upload_files +from fmu.sumo.uploader._logger import get_uploader_logger + # pylint: disable=C0103 # allow non-snake case variable names -logger = logging.getLogger(__name__) -logger.setLevel(logging.CRITICAL) +logger = get_uploader_logger() class SumoCase: - def __init__(self, case_metadata: str, sumo_connection, verbosity): + def __init__(self, case_metadata: str, sumo_connection, verbosity="WARNING"): logger.setLevel(verbosity) self.sumo_connection = sumo_connection self.case_metadata = _sanitize_datetimes(case_metadata) diff --git a/src/fmu/sumo/uploader/_sumofile.py b/src/fmu/sumo/uploader/_sumofile.py index 101101f..df19b8c 100644 --- a/src/fmu/sumo/uploader/_sumofile.py +++ b/src/fmu/sumo/uploader/_sumofile.py @@ -12,11 +12,12 @@ import logging import warnings import httpx +from fmu.sumo.uploader._logger import get_uploader_logger + # pylint: disable=C0103 # allow non-snake case variable names -logger = logging.getLogger(__name__) -logger.setLevel(logging.WARNING) +logger = get_uploader_logger() def _get_segyimport_cmdstr(blob_url, object_id, file_path, sample_unit): diff --git a/src/fmu/sumo/uploader/caseondisk.py b/src/fmu/sumo/uploader/caseondisk.py index 2c955e2..17f8810 100644 --- a/src/fmu/sumo/uploader/caseondisk.py +++ b/src/fmu/sumo/uploader/caseondisk.py @@ -16,10 +16,11 @@ from fmu.sumo.uploader._fileondisk import FileOnDisk from fmu.dataio import ExportData from fmu.dataio._utils import read_parameters_txt +from fmu.sumo.uploader._logger import get_uploader_logger -logger = logging.getLogger(__name__) -logger.setLevel(logging.CRITICAL) + +logger = get_uploader_logger() # pylint: disable=C0103 # allow non-snake case variable names @@ -64,7 +65,7 @@ class CaseOnDisk(SumoCase): """ def __init__( - self, case_metadata_path: str, sumo_connection, verbosity=logging.INFO + self, case_metadata_path: str, sumo_connection, verbosity=logging.WARNING ): """Initialize CaseOnDisk. diff --git a/src/fmu/sumo/uploader/caseonjob.py b/src/fmu/sumo/uploader/caseonjob.py index 9bdfc90..025b0a1 100644 --- a/src/fmu/sumo/uploader/caseonjob.py +++ b/src/fmu/sumo/uploader/caseonjob.py @@ -5,9 +5,10 @@ from fmu.sumo.uploader._sumocase import SumoCase from fmu.sumo.uploader._fileonjob import FileOnJob +from fmu.sumo.uploader._logger import get_uploader_logger -logger = logging.getLogger(__name__) -logger.setLevel(logging.CRITICAL) + +logger = get_uploader_logger() # pylint: disable=C0103 # allow non-snake case variable names diff --git a/src/fmu/sumo/uploader/scripts/sumo_upload.py b/src/fmu/sumo/uploader/scripts/sumo_upload.py index 68d9b87..02b3e08 100644 --- a/src/fmu/sumo/uploader/scripts/sumo_upload.py +++ b/src/fmu/sumo/uploader/scripts/sumo_upload.py @@ -16,9 +16,10 @@ from res.job_queue import ErtScript # type: ignore from fmu.sumo import uploader +from fmu.sumo.uploader._logger import get_uploader_logger + +logger = get_uploader_logger() -logger = logging.getLogger(__name__) -logger.setLevel(logging.WARNING) # This documentation is for sumo_uploader as an ERT workflow DESCRIPTION = """SUMO_UPLOAD will upload files to Sumo. The typical use case @@ -74,6 +75,8 @@ def main() -> None: parser = _get_parser() args = parser.parse_args() + logger.setLevel(logging.INFO) + if args.verbose: logger.setLevel(logging.INFO) if args.debug: @@ -92,6 +95,7 @@ def main() -> None: metadata_path=args.metadata_path, threads=args.threads, config_path=args.config_path, + verbosity=logging.INFO ) @@ -102,9 +106,11 @@ def sumo_upload_main( metadata_path: str, threads: int, config_path: str = "fmuconfig/output/global_variables.yml", + verbosity: int = logging.INFO ) -> None: """A "main" function that can be used both from command line and from ERT workflow""" + logger.setLevel(verbosity) logger.debug("Running fmu_uploader_main()") # Catch-all to ensure FMU workflow keeps running even if something happens. @@ -123,6 +129,7 @@ def sumo_upload_main( e = uploader.CaseOnDisk( case_metadata_path=case_metadata_path, sumo_connection=sumo_connection, + verbosity=verbosity ) # add files to the case on disk object logger.info("Adding files. Search path is %s", searchpath) @@ -163,6 +170,9 @@ def run(self, *args): # pylint: disable=no-self-use """Parse with a simplified command line parser, for ERT only, call sumo_upload_main()""" + + logger.setLevel(logging.WARNING) + logger.debug("Calling run() on SumoUpload") parser = _get_parser() args = parser.parse_args(args) @@ -174,6 +184,7 @@ def run(self, *args): metadata_path=args.metadata_path, threads=args.threads, config_path=args.config_path, + verbosity=logging.WARNING )