diff --git a/.github/workflows/linting_and_formatting.yaml b/.github/workflows/linting_and_formatting.yaml new file mode 100644 index 00000000..a2f678f2 --- /dev/null +++ b/.github/workflows/linting_and_formatting.yaml @@ -0,0 +1,27 @@ +name: Check formatting and linting + +on: + pull_request: + push: { branches: [main] } + +jobs: + ruff-check: + name: Run ruff lint and format checks + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Installing dependencies + run: pip install ruff + + - name: Run ruff lint + run: ruff check . + + - name: Run ruff format + run: ruff format . --check diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..1c7b1cfc --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +repos: +- repo: local + hooks: + - id: lint + name: Ruff Lint + description: Linting using ruff + entry: bash -c 'ruff check .' + language: system + stages: ["pre-commit", "pre-push"] + + - id: format + name: Ruff Format + description: Formatting using ruff + entry: bash -c 'ruff format . --check' + language: system + stages: ["pre-commit", "pre-push"] diff --git a/pyproject.toml b/pyproject.toml index fe93b7ff..47795324 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,14 +2,6 @@ requires = ["setuptools", "setuptools-scm"] build-backend = "setuptools.build_meta" -[tool.setuptools_scm] - -[tool.isort] -profile = "black" - -[tool.black] -line-length = 79 - [project] name = "fmu-sumo-sim2sumo" requires-python = ">=3.9" @@ -26,7 +18,7 @@ dependencies = [ [project.optional-dependencies] test = ["pytest"] -dev = ["pytest", "black", "flake8"] +dev = ["pytest", "ruff", "pre-commit"] nokomodo = ["ert"] docs = [ @@ -46,3 +38,25 @@ sim2sumo = "fmu.sumo.sim2sumo.main:main" [project.entry-points.ert] fmu_sumo_sim2sumo_jobs = "fmu.sumo.sim2sumo.hook_implementations.jobs" + +[tool.ruff] +exclude = [".env", ".git", ".github", ".venv", "venv"] + +line-length = 79 + +[tool.ruff.lint] +ignore = ["E501", "N802"] + +extend-select = [ + "C4", # Flake8-comprehensions + "I", # isort + "SIM", # Flake8-simplify + "TC", # Flake8-type-checking + "TID", # Flake8-tidy-imports + "N", # pep8-naming + "PD", # Pandas + "NPY", # NumPy +] + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401"] diff --git a/src/fmu/sumo/sim2sumo/_special_treatments.py b/src/fmu/sumo/sim2sumo/_special_treatments.py index 2c9ccdc3..e5630cea 100644 --- a/src/fmu/sumo/sim2sumo/_special_treatments.py +++ b/src/fmu/sumo/sim2sumo/_special_treatments.py @@ -1,5 +1,6 @@ """Special treatment of some options used in res2df""" +import contextlib import importlib import logging from inspect import signature @@ -73,7 +74,7 @@ def find_functions_and_docstring(submod): "extract": func, "options": tuple( name - for name in signature(func).parameters.keys() + for name in signature(func).parameters if name not in {"deck", "eclfiles"} ), "arrow_convertor": find_arrow_convertor(import_path), @@ -101,10 +102,8 @@ def _define_submodules(): except AttributeError: submod_string = "vfp._vfp" submod = "vfp" - try: + with contextlib.suppress(AttributeError): submodules[submod] = find_functions_and_docstring(submod_string) - except AttributeError: - pass # No df function in submod_path, skip it return tuple(submodules.keys()), submodules @@ -128,7 +127,7 @@ def tidy(frame): ) unwanted_posix.unlink() if "WELLETC" in frame.columns: - frame.drop(["WELLETC"], axis=1, inplace=True) + frame = frame.drop(["WELLETC"], axis=1) return frame @@ -151,10 +150,7 @@ def vfp_to_arrow_dict(datafile, options): vfp_dict = {} keyword = options.get("keyword", ["VFPPROD", "VFPINJ"]) vfpnumbers = options.get("vfpnumbers", None) - if isinstance(keyword, str): - keywords = [keyword] - else: - keywords = keyword + keywords = [keyword] if isinstance(keyword, str) else keyword for keyword in keywords: vfp_dict[keyword] = res2df.vfp._vfp.pyarrow_tables( diff --git a/src/fmu/sumo/sim2sumo/common.py b/src/fmu/sumo/sim2sumo/common.py index 6f0b10aa..d410c30c 100644 --- a/src/fmu/sumo/sim2sumo/common.py +++ b/src/fmu/sumo/sim2sumo/common.py @@ -6,16 +6,15 @@ import psutil import yaml +from res2df.common import convert_lyrlist_to_zonemap, parse_lyrfile from fmu.dataio import ExportData -from fmu.sumo.uploader import SumoConnection -from fmu.sumo.uploader._upload_files import upload_files from fmu.sumo.sim2sumo._special_treatments import ( SUBMOD_DICT, SUBMODULES, ) - -from res2df.common import convert_lyrlist_to_zonemap, parse_lyrfile +from fmu.sumo.uploader import SumoConnection +from fmu.sumo.uploader._upload_files import upload_files def yaml_load(file_name): @@ -123,34 +122,26 @@ def find_datafiles(seedpoint=None): datafiles.append(full_path) else: datafiles.extend( - [ - f - for f in full_path.parent.rglob( - f"{full_path.name}" - ) - ] + list(full_path.parent.rglob(f"{full_path.name}")) ) else: for filetype in valid_filetypes: if not full_path.is_dir(): # Search for valid files within the directory datafiles.extend( - [ - f - for f in full_path.parent.rglob( + list( + full_path.parent.rglob( f"{full_path.name}*{filetype}" ) - ] + ) ) else: # Search for valid files within the directory - datafiles.extend( - [f for f in full_path.rglob(f"*{filetype}")] - ) + datafiles.extend(list(full_path.rglob(f"*{filetype}"))) else: # Search the current working directory if no seedpoint is provided for filetype in valid_filetypes: - datafiles.extend([f for f in cwd.rglob(f"*/*/*{filetype}")]) + datafiles.extend(list(cwd.rglob(f"*/*/*{filetype}"))) # Filter out files with duplicate stems, keeping the first occurrence unique_stems = set() unique_datafiles = [] @@ -327,10 +318,7 @@ def find_datefield(text_string): str| None: date as string or None """ datesearch = re.search(".*_([0-9]{8})$", text_string) - if datesearch is not None: - date = datesearch.group(1) - else: - date = None + date = datesearch.group(1) if datesearch is not None else None return date diff --git a/src/fmu/sumo/sim2sumo/forward_models/__init__.py b/src/fmu/sumo/sim2sumo/forward_models/__init__.py index 4808c230..8652bd80 100644 --- a/src/fmu/sumo/sim2sumo/forward_models/__init__.py +++ b/src/fmu/sumo/sim2sumo/forward_models/__init__.py @@ -1,4 +1,5 @@ import subprocess + from ert import ( ForwardModelStepJSON, ForwardModelStepPlugin, diff --git a/src/fmu/sumo/sim2sumo/grid3d.py b/src/fmu/sumo/sim2sumo/grid3d.py index 724091fa..19774d82 100755 --- a/src/fmu/sumo/sim2sumo/grid3d.py +++ b/src/fmu/sumo/sim2sumo/grid3d.py @@ -1,24 +1,26 @@ #!/usr/bin/env python """Upload grid3d data from reservoir simulators to Sumo - Does three things: - 1. Extracts data from simulator to roff files - 2. Adds the required metadata while exporting to disc - 3. Uploads to Sumo +Does three things: +1. Extracts data from simulator to roff files +2. Adds the required metadata while exporting to disc +3. Uploads to Sumo """ + import logging -from pathlib import Path from datetime import datetime - from io import BytesIO +from pathlib import Path + import numpy as np from resdata.grid import Grid from resdata.resfile import ResdataRestartFile from xtgeo import GridProperty, grid_from_file from xtgeo.grid3d import _gridprop_import_eclrun as eclrun from xtgeo.io._file import FileWrapper -from fmu.sumo.uploader._fileonjob import FileOnJob from fmu.dataio import ExportData +from fmu.sumo.uploader._fileonjob import FileOnJob + from .common import find_datefield, give_name @@ -59,10 +61,7 @@ def generate_grid3d_meta(datafile, obj, prefix, config): else: content = {"property": {"is_discrete": False}} - if prefix == "grid": - name = prefix - else: - name = f"{prefix}-{obj.name}" + name = prefix if prefix == "grid" else f"{prefix}-{obj.name}" tagname = give_name(datafile) exp_args = { "config": config, diff --git a/src/fmu/sumo/sim2sumo/hook_implementations/jobs.py b/src/fmu/sumo/sim2sumo/hook_implementations/jobs.py index 05fb68af..75b4cbb1 100644 --- a/src/fmu/sumo/sim2sumo/hook_implementations/jobs.py +++ b/src/fmu/sumo/sim2sumo/hook_implementations/jobs.py @@ -74,9 +74,7 @@ def _get_jobs_from_directory(directory): # pylint: disable=no-value-for-parameter @hook_implementation -@plugin_response( - plugin_name=PLUGIN_NAME -) # pylint: disable=no-value-for-parameter +@plugin_response(plugin_name=PLUGIN_NAME) # pylint: disable=no-value-for-parameter def installable_jobs(): """Return installable jobs @@ -87,9 +85,7 @@ def installable_jobs(): @hook_implementation -@plugin_response( - plugin_name=PLUGIN_NAME -) # pylint: disable=no-value-for-parameter +@plugin_response(plugin_name=PLUGIN_NAME) # pylint: disable=no-value-for-parameter def job_documentation(job_name): sumo_fmu_jobs = set(installable_jobs().data.keys()) if job_name not in sumo_fmu_jobs: diff --git a/src/fmu/sumo/sim2sumo/main.py b/src/fmu/sumo/sim2sumo/main.py index 10e90028..dd57335a 100644 --- a/src/fmu/sumo/sim2sumo/main.py +++ b/src/fmu/sumo/sim2sumo/main.py @@ -2,11 +2,12 @@ import argparse import logging +import sys from os import environ +from .common import Dispatcher, create_config_dict, yaml_load from .grid3d import upload_simulation_runs from .tables import upload_tables -from .common import yaml_load, Dispatcher, create_config_dict def parse_args(): @@ -52,9 +53,9 @@ def main(): logger = logging.getLogger(__file__ + ".main") missing = [] - for envVar in REQUIRED_ENV_VARS: - if envVar not in environ: - missing.append(envVar) + for env_var in REQUIRED_ENV_VARS: + if env_var not in environ: + missing.append(env_var) if missing: print( @@ -63,7 +64,7 @@ def main(): "This can happen if sim2sumo was called outside the ERT context.\n" "Stopping." ) - exit() + sys.exit() args = parse_args() diff --git a/src/fmu/sumo/sim2sumo/tables.py b/src/fmu/sumo/sim2sumo/tables.py index 731af3ea..ed9ed6e8 100644 --- a/src/fmu/sumo/sim2sumo/tables.py +++ b/src/fmu/sumo/sim2sumo/tables.py @@ -1,35 +1,34 @@ """Upload tabular data from reservoir simulators to sumo - Does three things: - 1. Extracts data from simulator to arrow files - 2. Adds the required metadata while exporting to disc - 3. Uploads to Sumo +Does three things: +1. Extracts data from simulator to arrow files +2. Adds the required metadata while exporting to disc +3. Uploads to Sumo """ import logging import sys +from pathlib import Path from typing import Union +import pandas as pd import pyarrow as pa import pyarrow.parquet as pq -import pandas as pd import res2df + +from fmu.dataio import ExportData from fmu.sumo.uploader._fileonjob import FileOnJob from ._special_treatments import ( SUBMOD_DICT, - tidy, convert_to_arrow, + tidy, vfp_to_arrow_dict, ) - -from pathlib import Path -from fmu.dataio import ExportData from .common import ( find_datefield, give_name, ) - SUBMOD_CONTENT = { "summary": "timeseries", "satfunc": "relperm", @@ -140,12 +139,10 @@ def get_table( logger = logging.getLogger(__file__ + ".get_table") extract_df = SUBMOD_DICT[submod]["extract"] arrow = kwargs.get("arrow", True) - try: - del kwargs[ - "arrow" - ] # This argument should not be passed to extract function - except KeyError: - pass # No arrow key to delete + from contextlib import suppress + + with suppress(KeyError): + del kwargs["arrow"] output = None try: logger.info( diff --git a/tests/conftest.py b/tests/conftest.py index c5d33fee..73e08b29 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,19 +1,19 @@ import os import shutil -import pandas as pd +import uuid from datetime import datetime from pathlib import Path -import uuid +import pandas as pd import pytest import yaml -from fmu.config.utilities import yaml_load -from fmu.sumo.uploader import CaseOnDisk from httpx import HTTPStatusError from sumo.wrapper import SumoClient - from xtgeo import grid_from_file, gridproperty_from_file + +from fmu.config.utilities import yaml_load from fmu.sumo.sim2sumo._special_treatments import convert_to_arrow +from fmu.sumo.uploader import CaseOnDisk REEK_ROOT = Path(__file__).parent / "data/reek" REEK_REAL0 = REEK_ROOT / "realization-0/iter-0/" diff --git a/tests/test_functions.py b/tests/test_functions.py index 796eb64c..1801d345 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -1,38 +1,35 @@ """Test utility ecl2csv""" import os -from numpy.ma import allclose, allequal +from io import BytesIO from shutil import copytree from time import sleep -from io import BytesIO + import pandas as pd import pyarrow as pa import pyarrow.parquet as pq import pytest - +from conftest import REEK_DATA_FILE, REEK_REAL0, REEK_REAL1 +from numpy.ma import allclose, allequal from xtgeo import GridProperty, gridproperty_from_file from fmu.sumo.sim2sumo import grid3d, tables +from fmu.sumo.sim2sumo._special_treatments import ( + SUBMODULES, + _define_submodules, + convert_to_arrow, +) from fmu.sumo.sim2sumo.common import ( - find_datafiles, - create_config_dict, - nodisk_upload, Dispatcher, - find_datefield, + create_config_dict, filter_options, + find_datafiles, + find_datefield, get_case_uuid, -) -from fmu.sumo.sim2sumo._special_treatments import ( - _define_submodules, - convert_to_arrow, - SUBMODULES, + nodisk_upload, ) from fmu.sumo.uploader import SumoConnection - -from conftest import REEK_REAL0, REEK_REAL1, REEK_DATA_FILE - - SLEEP_TIME = 3 @@ -307,7 +304,7 @@ def test_submodules_dict(): ), f"Left part of folder path for {submod_name}" assert isinstance(submod_dict, dict), f"{submod_name} has no subdict" assert ( - "options" in submod_dict.keys() + "options" in submod_dict ), f"{submod_name} does not have any options" assert isinstance( diff --git a/tests/test_w_drogon.py b/tests/test_w_drogon.py index d61c890f..4ac4607e 100644 --- a/tests/test_w_drogon.py +++ b/tests/test_w_drogon.py @@ -1,12 +1,13 @@ from pathlib import Path + +import pytest +from test_functions import check_sumo + from fmu.sumo.sim2sumo._special_treatments import vfp_to_arrow_dict +from fmu.sumo.sim2sumo.common import Dispatcher from fmu.sumo.sim2sumo.tables import ( upload_vfp_tables_from_simulation_run, - get_table, ) -from fmu.sumo.sim2sumo.common import Dispatcher -from test_functions import check_sumo -import pytest DROGON = Path(__file__).parent / "data/drogon/" DROGON_REAL = DROGON / "realization-0/iter-0/" diff --git a/tests/test_with_ert.py b/tests/test_with_ert.py index fe3b5111..9a170169 100644 --- a/tests/test_with_ert.py +++ b/tests/test_with_ert.py @@ -3,7 +3,6 @@ # to exist etc. Tests that run ERT should therefore create their own # temporary file structure, completely separate from other tests. from pathlib import Path - from subprocess import PIPE, Popen @@ -13,7 +12,6 @@ def write_ert_config_and_run(runpath): ert_full_config_path = runpath / ert_config_path print(f"Running with path {ert_full_config_path}") with open(ert_full_config_path, "w", encoding=encoding) as stream: - stream.write( ( "DEFINE dev\nNUM_REALIZATIONS 1\nMAX_SUBMIT"