From 74c4684dd65559e3ebd70f53942d1d03aae560c7 Mon Sep 17 00:00:00 2001 From: Sean Sinclair <146738689+sean-sinclair@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:21:16 +0100 Subject: [PATCH 1/6] build(pyproject): replace black and flake8 with Ruff --- pyproject.toml | 43 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fe93b7f..005fe5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,14 +2,6 @@ requires = ["setuptools", "setuptools-scm"] build-backend = "setuptools.build_meta" -[tool.setuptools_scm] - -[tool.isort] -profile = "black" - -[tool.black] -line-length = 79 - [project] name = "fmu-sumo-sim2sumo" requires-python = ">=3.9" @@ -26,7 +18,7 @@ dependencies = [ [project.optional-dependencies] test = ["pytest"] -dev = ["pytest", "black", "flake8"] +dev = ["pytest", "ruff"] nokomodo = ["ert"] docs = [ @@ -46,3 +38,36 @@ sim2sumo = "fmu.sumo.sim2sumo.main:main" [project.entry-points.ert] fmu_sumo_sim2sumo_jobs = "fmu.sumo.sim2sumo.hook_implementations.jobs" + +[tool.ruff] +exclude = [ + ".env", + ".git", + ".github", + ".venv", + "venv", +] + +line-length = 79 + +[tool.ruff.lint] +ignore = [ + "E501", + "N802", +] + +extend-select = [ + "C4", # Flake8-comprehensions + "I", # isort + "SIM", # Flake8-simplify + "TC", # Flake8-type-checking + "TID", # Flake8-tidy-imports + "N", # pep8-naming + "PD", # Pandas + "NPY", # NumPy +] + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401"] + + From 5579744133ab5061b36a9105f578dbdaab76c3cb Mon Sep 17 00:00:00 2001 From: Sean Sinclair <146738689+sean-sinclair@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:26:29 +0100 Subject: [PATCH 2/6] style: Automatic format and lint fixes --- src/fmu/sumo/sim2sumo/common.py | 7 ++--- .../sumo/sim2sumo/forward_models/__init__.py | 1 + src/fmu/sumo/sim2sumo/grid3d.py | 16 +++++----- .../sim2sumo/hook_implementations/jobs.py | 8 ++--- src/fmu/sumo/sim2sumo/main.py | 2 +- src/fmu/sumo/sim2sumo/tables.py | 19 ++++++------ tests/conftest.py | 10 +++---- tests/test_functions.py | 29 +++++++++---------- tests/test_w_drogon.py | 9 +++--- tests/test_with_ert.py | 2 -- 10 files changed, 48 insertions(+), 55 deletions(-) diff --git a/src/fmu/sumo/sim2sumo/common.py b/src/fmu/sumo/sim2sumo/common.py index 6f0b10a..9adc957 100644 --- a/src/fmu/sumo/sim2sumo/common.py +++ b/src/fmu/sumo/sim2sumo/common.py @@ -6,16 +6,15 @@ import psutil import yaml +from res2df.common import convert_lyrlist_to_zonemap, parse_lyrfile from fmu.dataio import ExportData -from fmu.sumo.uploader import SumoConnection -from fmu.sumo.uploader._upload_files import upload_files from fmu.sumo.sim2sumo._special_treatments import ( SUBMOD_DICT, SUBMODULES, ) - -from res2df.common import convert_lyrlist_to_zonemap, parse_lyrfile +from fmu.sumo.uploader import SumoConnection +from fmu.sumo.uploader._upload_files import upload_files def yaml_load(file_name): diff --git a/src/fmu/sumo/sim2sumo/forward_models/__init__.py b/src/fmu/sumo/sim2sumo/forward_models/__init__.py index 4808c23..8652bd8 100644 --- a/src/fmu/sumo/sim2sumo/forward_models/__init__.py +++ b/src/fmu/sumo/sim2sumo/forward_models/__init__.py @@ -1,4 +1,5 @@ import subprocess + from ert import ( ForwardModelStepJSON, ForwardModelStepPlugin, diff --git a/src/fmu/sumo/sim2sumo/grid3d.py b/src/fmu/sumo/sim2sumo/grid3d.py index 724091f..1dfa9a3 100755 --- a/src/fmu/sumo/sim2sumo/grid3d.py +++ b/src/fmu/sumo/sim2sumo/grid3d.py @@ -1,24 +1,26 @@ #!/usr/bin/env python """Upload grid3d data from reservoir simulators to Sumo - Does three things: - 1. Extracts data from simulator to roff files - 2. Adds the required metadata while exporting to disc - 3. Uploads to Sumo +Does three things: +1. Extracts data from simulator to roff files +2. Adds the required metadata while exporting to disc +3. Uploads to Sumo """ + import logging -from pathlib import Path from datetime import datetime - from io import BytesIO +from pathlib import Path + import numpy as np from resdata.grid import Grid from resdata.resfile import ResdataRestartFile from xtgeo import GridProperty, grid_from_file from xtgeo.grid3d import _gridprop_import_eclrun as eclrun from xtgeo.io._file import FileWrapper -from fmu.sumo.uploader._fileonjob import FileOnJob from fmu.dataio import ExportData +from fmu.sumo.uploader._fileonjob import FileOnJob + from .common import find_datefield, give_name diff --git a/src/fmu/sumo/sim2sumo/hook_implementations/jobs.py b/src/fmu/sumo/sim2sumo/hook_implementations/jobs.py index 05fb68a..75b4cbb 100644 --- a/src/fmu/sumo/sim2sumo/hook_implementations/jobs.py +++ b/src/fmu/sumo/sim2sumo/hook_implementations/jobs.py @@ -74,9 +74,7 @@ def _get_jobs_from_directory(directory): # pylint: disable=no-value-for-parameter @hook_implementation -@plugin_response( - plugin_name=PLUGIN_NAME -) # pylint: disable=no-value-for-parameter +@plugin_response(plugin_name=PLUGIN_NAME) # pylint: disable=no-value-for-parameter def installable_jobs(): """Return installable jobs @@ -87,9 +85,7 @@ def installable_jobs(): @hook_implementation -@plugin_response( - plugin_name=PLUGIN_NAME -) # pylint: disable=no-value-for-parameter +@plugin_response(plugin_name=PLUGIN_NAME) # pylint: disable=no-value-for-parameter def job_documentation(job_name): sumo_fmu_jobs = set(installable_jobs().data.keys()) if job_name not in sumo_fmu_jobs: diff --git a/src/fmu/sumo/sim2sumo/main.py b/src/fmu/sumo/sim2sumo/main.py index 10e9002..4257218 100644 --- a/src/fmu/sumo/sim2sumo/main.py +++ b/src/fmu/sumo/sim2sumo/main.py @@ -4,9 +4,9 @@ import logging from os import environ +from .common import Dispatcher, create_config_dict, yaml_load from .grid3d import upload_simulation_runs from .tables import upload_tables -from .common import yaml_load, Dispatcher, create_config_dict def parse_args(): diff --git a/src/fmu/sumo/sim2sumo/tables.py b/src/fmu/sumo/sim2sumo/tables.py index 731af3e..9438e64 100644 --- a/src/fmu/sumo/sim2sumo/tables.py +++ b/src/fmu/sumo/sim2sumo/tables.py @@ -1,35 +1,34 @@ """Upload tabular data from reservoir simulators to sumo - Does three things: - 1. Extracts data from simulator to arrow files - 2. Adds the required metadata while exporting to disc - 3. Uploads to Sumo +Does three things: +1. Extracts data from simulator to arrow files +2. Adds the required metadata while exporting to disc +3. Uploads to Sumo """ import logging import sys +from pathlib import Path from typing import Union +import pandas as pd import pyarrow as pa import pyarrow.parquet as pq -import pandas as pd import res2df + +from fmu.dataio import ExportData from fmu.sumo.uploader._fileonjob import FileOnJob from ._special_treatments import ( SUBMOD_DICT, - tidy, convert_to_arrow, + tidy, vfp_to_arrow_dict, ) - -from pathlib import Path -from fmu.dataio import ExportData from .common import ( find_datefield, give_name, ) - SUBMOD_CONTENT = { "summary": "timeseries", "satfunc": "relperm", diff --git a/tests/conftest.py b/tests/conftest.py index c5d33fe..73e08b2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,19 +1,19 @@ import os import shutil -import pandas as pd +import uuid from datetime import datetime from pathlib import Path -import uuid +import pandas as pd import pytest import yaml -from fmu.config.utilities import yaml_load -from fmu.sumo.uploader import CaseOnDisk from httpx import HTTPStatusError from sumo.wrapper import SumoClient - from xtgeo import grid_from_file, gridproperty_from_file + +from fmu.config.utilities import yaml_load from fmu.sumo.sim2sumo._special_treatments import convert_to_arrow +from fmu.sumo.uploader import CaseOnDisk REEK_ROOT = Path(__file__).parent / "data/reek" REEK_REAL0 = REEK_ROOT / "realization-0/iter-0/" diff --git a/tests/test_functions.py b/tests/test_functions.py index 796eb64..454be28 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -1,38 +1,35 @@ """Test utility ecl2csv""" import os -from numpy.ma import allclose, allequal +from io import BytesIO from shutil import copytree from time import sleep -from io import BytesIO + import pandas as pd import pyarrow as pa import pyarrow.parquet as pq import pytest - +from conftest import REEK_DATA_FILE, REEK_REAL0, REEK_REAL1 +from numpy.ma import allclose, allequal from xtgeo import GridProperty, gridproperty_from_file from fmu.sumo.sim2sumo import grid3d, tables +from fmu.sumo.sim2sumo._special_treatments import ( + SUBMODULES, + _define_submodules, + convert_to_arrow, +) from fmu.sumo.sim2sumo.common import ( - find_datafiles, - create_config_dict, - nodisk_upload, Dispatcher, - find_datefield, + create_config_dict, filter_options, + find_datafiles, + find_datefield, get_case_uuid, -) -from fmu.sumo.sim2sumo._special_treatments import ( - _define_submodules, - convert_to_arrow, - SUBMODULES, + nodisk_upload, ) from fmu.sumo.uploader import SumoConnection - -from conftest import REEK_REAL0, REEK_REAL1, REEK_DATA_FILE - - SLEEP_TIME = 3 diff --git a/tests/test_w_drogon.py b/tests/test_w_drogon.py index d61c890..4ac4607 100644 --- a/tests/test_w_drogon.py +++ b/tests/test_w_drogon.py @@ -1,12 +1,13 @@ from pathlib import Path + +import pytest +from test_functions import check_sumo + from fmu.sumo.sim2sumo._special_treatments import vfp_to_arrow_dict +from fmu.sumo.sim2sumo.common import Dispatcher from fmu.sumo.sim2sumo.tables import ( upload_vfp_tables_from_simulation_run, - get_table, ) -from fmu.sumo.sim2sumo.common import Dispatcher -from test_functions import check_sumo -import pytest DROGON = Path(__file__).parent / "data/drogon/" DROGON_REAL = DROGON / "realization-0/iter-0/" diff --git a/tests/test_with_ert.py b/tests/test_with_ert.py index fe3b511..9a17016 100644 --- a/tests/test_with_ert.py +++ b/tests/test_with_ert.py @@ -3,7 +3,6 @@ # to exist etc. Tests that run ERT should therefore create their own # temporary file structure, completely separate from other tests. from pathlib import Path - from subprocess import PIPE, Popen @@ -13,7 +12,6 @@ def write_ert_config_and_run(runpath): ert_full_config_path = runpath / ert_config_path print(f"Running with path {ert_full_config_path}") with open(ert_full_config_path, "w", encoding=encoding) as stream: - stream.write( ( "DEFINE dev\nNUM_REALIZATIONS 1\nMAX_SUBMIT" From e3d9ad62c12faa30bc550b84624813253fdc47d7 Mon Sep 17 00:00:00 2001 From: Sean Sinclair <146738689+sean-sinclair@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:54:09 +0100 Subject: [PATCH 3/6] style: Manual ruff lint modifications --- src/fmu/sumo/sim2sumo/_special_treatments.py | 14 ++++------- src/fmu/sumo/sim2sumo/common.py | 25 ++++++-------------- src/fmu/sumo/sim2sumo/grid3d.py | 5 +--- src/fmu/sumo/sim2sumo/main.py | 9 +++---- src/fmu/sumo/sim2sumo/tables.py | 10 ++++---- tests/test_functions.py | 2 +- 6 files changed, 23 insertions(+), 42 deletions(-) diff --git a/src/fmu/sumo/sim2sumo/_special_treatments.py b/src/fmu/sumo/sim2sumo/_special_treatments.py index 2c9ccdc..e5630ce 100644 --- a/src/fmu/sumo/sim2sumo/_special_treatments.py +++ b/src/fmu/sumo/sim2sumo/_special_treatments.py @@ -1,5 +1,6 @@ """Special treatment of some options used in res2df""" +import contextlib import importlib import logging from inspect import signature @@ -73,7 +74,7 @@ def find_functions_and_docstring(submod): "extract": func, "options": tuple( name - for name in signature(func).parameters.keys() + for name in signature(func).parameters if name not in {"deck", "eclfiles"} ), "arrow_convertor": find_arrow_convertor(import_path), @@ -101,10 +102,8 @@ def _define_submodules(): except AttributeError: submod_string = "vfp._vfp" submod = "vfp" - try: + with contextlib.suppress(AttributeError): submodules[submod] = find_functions_and_docstring(submod_string) - except AttributeError: - pass # No df function in submod_path, skip it return tuple(submodules.keys()), submodules @@ -128,7 +127,7 @@ def tidy(frame): ) unwanted_posix.unlink() if "WELLETC" in frame.columns: - frame.drop(["WELLETC"], axis=1, inplace=True) + frame = frame.drop(["WELLETC"], axis=1) return frame @@ -151,10 +150,7 @@ def vfp_to_arrow_dict(datafile, options): vfp_dict = {} keyword = options.get("keyword", ["VFPPROD", "VFPINJ"]) vfpnumbers = options.get("vfpnumbers", None) - if isinstance(keyword, str): - keywords = [keyword] - else: - keywords = keyword + keywords = [keyword] if isinstance(keyword, str) else keyword for keyword in keywords: vfp_dict[keyword] = res2df.vfp._vfp.pyarrow_tables( diff --git a/src/fmu/sumo/sim2sumo/common.py b/src/fmu/sumo/sim2sumo/common.py index 9adc957..d410c30 100644 --- a/src/fmu/sumo/sim2sumo/common.py +++ b/src/fmu/sumo/sim2sumo/common.py @@ -122,34 +122,26 @@ def find_datafiles(seedpoint=None): datafiles.append(full_path) else: datafiles.extend( - [ - f - for f in full_path.parent.rglob( - f"{full_path.name}" - ) - ] + list(full_path.parent.rglob(f"{full_path.name}")) ) else: for filetype in valid_filetypes: if not full_path.is_dir(): # Search for valid files within the directory datafiles.extend( - [ - f - for f in full_path.parent.rglob( + list( + full_path.parent.rglob( f"{full_path.name}*{filetype}" ) - ] + ) ) else: # Search for valid files within the directory - datafiles.extend( - [f for f in full_path.rglob(f"*{filetype}")] - ) + datafiles.extend(list(full_path.rglob(f"*{filetype}"))) else: # Search the current working directory if no seedpoint is provided for filetype in valid_filetypes: - datafiles.extend([f for f in cwd.rglob(f"*/*/*{filetype}")]) + datafiles.extend(list(cwd.rglob(f"*/*/*{filetype}"))) # Filter out files with duplicate stems, keeping the first occurrence unique_stems = set() unique_datafiles = [] @@ -326,10 +318,7 @@ def find_datefield(text_string): str| None: date as string or None """ datesearch = re.search(".*_([0-9]{8})$", text_string) - if datesearch is not None: - date = datesearch.group(1) - else: - date = None + date = datesearch.group(1) if datesearch is not None else None return date diff --git a/src/fmu/sumo/sim2sumo/grid3d.py b/src/fmu/sumo/sim2sumo/grid3d.py index 1dfa9a3..19774d8 100755 --- a/src/fmu/sumo/sim2sumo/grid3d.py +++ b/src/fmu/sumo/sim2sumo/grid3d.py @@ -61,10 +61,7 @@ def generate_grid3d_meta(datafile, obj, prefix, config): else: content = {"property": {"is_discrete": False}} - if prefix == "grid": - name = prefix - else: - name = f"{prefix}-{obj.name}" + name = prefix if prefix == "grid" else f"{prefix}-{obj.name}" tagname = give_name(datafile) exp_args = { "config": config, diff --git a/src/fmu/sumo/sim2sumo/main.py b/src/fmu/sumo/sim2sumo/main.py index 4257218..dd57335 100644 --- a/src/fmu/sumo/sim2sumo/main.py +++ b/src/fmu/sumo/sim2sumo/main.py @@ -2,6 +2,7 @@ import argparse import logging +import sys from os import environ from .common import Dispatcher, create_config_dict, yaml_load @@ -52,9 +53,9 @@ def main(): logger = logging.getLogger(__file__ + ".main") missing = [] - for envVar in REQUIRED_ENV_VARS: - if envVar not in environ: - missing.append(envVar) + for env_var in REQUIRED_ENV_VARS: + if env_var not in environ: + missing.append(env_var) if missing: print( @@ -63,7 +64,7 @@ def main(): "This can happen if sim2sumo was called outside the ERT context.\n" "Stopping." ) - exit() + sys.exit() args = parse_args() diff --git a/src/fmu/sumo/sim2sumo/tables.py b/src/fmu/sumo/sim2sumo/tables.py index 9438e64..ed9ed6e 100644 --- a/src/fmu/sumo/sim2sumo/tables.py +++ b/src/fmu/sumo/sim2sumo/tables.py @@ -139,12 +139,10 @@ def get_table( logger = logging.getLogger(__file__ + ".get_table") extract_df = SUBMOD_DICT[submod]["extract"] arrow = kwargs.get("arrow", True) - try: - del kwargs[ - "arrow" - ] # This argument should not be passed to extract function - except KeyError: - pass # No arrow key to delete + from contextlib import suppress + + with suppress(KeyError): + del kwargs["arrow"] output = None try: logger.info( diff --git a/tests/test_functions.py b/tests/test_functions.py index 454be28..1801d34 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -304,7 +304,7 @@ def test_submodules_dict(): ), f"Left part of folder path for {submod_name}" assert isinstance(submod_dict, dict), f"{submod_name} has no subdict" assert ( - "options" in submod_dict.keys() + "options" in submod_dict ), f"{submod_name} does not have any options" assert isinstance( From b54bf6c73e6bbecc1f07d8e2c16d2eeb91861f29 Mon Sep 17 00:00:00 2001 From: Sean Sinclair <146738689+sean-sinclair@users.noreply.github.com> Date: Fri, 20 Dec 2024 08:38:07 +0100 Subject: [PATCH 4/6] ci: Ruff lint and format checks on PR --- .github/workflows/linting_and_formatting.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/linting_and_formatting.yaml diff --git a/.github/workflows/linting_and_formatting.yaml b/.github/workflows/linting_and_formatting.yaml new file mode 100644 index 0000000..a2f678f --- /dev/null +++ b/.github/workflows/linting_and_formatting.yaml @@ -0,0 +1,27 @@ +name: Check formatting and linting + +on: + pull_request: + push: { branches: [main] } + +jobs: + ruff-check: + name: Run ruff lint and format checks + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Installing dependencies + run: pip install ruff + + - name: Run ruff lint + run: ruff check . + + - name: Run ruff format + run: ruff format . --check From 374e910bc60fa8ee2df9dc48bbcc4f26699ac334 Mon Sep 17 00:00:00 2001 From: Sean Sinclair <146738689+sean-sinclair@users.noreply.github.com> Date: Fri, 20 Dec 2024 10:11:38 +0100 Subject: [PATCH 5/6] build(pre-commit): Ruff formatting and linting --- .pre-commit-config.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..1c7b1cf --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +repos: +- repo: local + hooks: + - id: lint + name: Ruff Lint + description: Linting using ruff + entry: bash -c 'ruff check .' + language: system + stages: ["pre-commit", "pre-push"] + + - id: format + name: Ruff Format + description: Formatting using ruff + entry: bash -c 'ruff format . --check' + language: system + stages: ["pre-commit", "pre-push"] From e8366a2209c3ca3bb1bbeb4f4b7c06730091bb97 Mon Sep 17 00:00:00 2001 From: Sean Sinclair <146738689+sean-sinclair@users.noreply.github.com> Date: Fri, 20 Dec 2024 11:22:41 +0100 Subject: [PATCH 6/6] build(pyproject): Pre-commit dev dependency --- pyproject.toml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 005fe5f..4779532 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ [project.optional-dependencies] test = ["pytest"] -dev = ["pytest", "ruff"] +dev = ["pytest", "ruff", "pre-commit"] nokomodo = ["ert"] docs = [ @@ -40,21 +40,12 @@ sim2sumo = "fmu.sumo.sim2sumo.main:main" fmu_sumo_sim2sumo_jobs = "fmu.sumo.sim2sumo.hook_implementations.jobs" [tool.ruff] -exclude = [ - ".env", - ".git", - ".github", - ".venv", - "venv", -] +exclude = [".env", ".git", ".github", ".venv", "venv"] line-length = 79 [tool.ruff.lint] -ignore = [ - "E501", - "N802", -] +ignore = ["E501", "N802"] extend-select = [ "C4", # Flake8-comprehensions @@ -64,10 +55,8 @@ extend-select = [ "TID", # Flake8-tidy-imports "N", # pep8-naming "PD", # Pandas - "NPY", # NumPy + "NPY", # NumPy ] [tool.ruff.lint.per-file-ignores] "__init__.py" = ["F401"] - -