From dfb873f602745c5d6c69f3f94586e756a644911f Mon Sep 17 00:00:00 2001 From: DanSava Date: Tue, 27 Feb 2024 17:57:00 +0200 Subject: [PATCH] Replace magic string in the runpath --- src/ert/enkf_main.py | 8 +-- src/ert/run_context.py | 1 + src/ert/run_models/base_run_model.py | 8 ++- src/ert/runpaths.py | 24 +++++--- .../workflows/export_runpath.py | 2 +- src/ert/simulator/simulation_context.py | 4 +- tests/unit_tests/config/test_defines.py | 2 +- tests/unit_tests/config/test_gen_kw_config.py | 2 +- .../run_models/test_base_run_model.py | 15 +++-- tests/unit_tests/storage/create_runpath.py | 2 +- tests/unit_tests/test_enkf_main.py | 4 +- tests/unit_tests/test_enkf_runpath.py | 8 +-- tests/unit_tests/test_load_forward_model.py | 6 +- tests/unit_tests/test_run_path_creation.py | 61 ++++++++++++++----- tests/unit_tests/test_runpaths.py | 4 +- tests/unit_tests/test_summary_response.py | 2 +- 16 files changed, 102 insertions(+), 51 deletions(-) diff --git a/src/ert/enkf_main.py b/src/ert/enkf_main.py index 106a1d513c2..1f58cc3e96c 100644 --- a/src/ert/enkf_main.py +++ b/src/ert/enkf_main.py @@ -4,7 +4,6 @@ import logging import os import time -from copy import copy from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, Dict, Iterable, List, Mapping, Optional, Union @@ -197,13 +196,10 @@ def sample_prior( def create_run_path( run_context: RunContext, - substitution_list: SubstitutionList, ert_config: ErtConfig, ) -> None: t = time.perf_counter() - substitution_list = copy(substitution_list) - substitution_list[""] = run_context.sim_fs.name - substitution_list[""] = run_context.sim_fs.name + substitution_list = ert_config.substitution_list for iens, run_arg in enumerate(run_context): run_path = Path(run_arg.runpath) if run_context.is_active(iens): @@ -279,7 +275,7 @@ def ensemble_context( jobname_format=jobname_format, runpath_format=runpath_format, filename=runpath_file, - substitute=substitution_list.substitute_real_iter, + substitution_list=substitution_list, ) return RunContext( sim_fs=case, diff --git a/src/ert/run_context.py b/src/ert/run_context.py index 9511a3035ec..79dd8ed383e 100644 --- a/src/ert/run_context.py +++ b/src/ert/run_context.py @@ -27,6 +27,7 @@ class RunContext: def __post_init__(self) -> None: self.run_id = uuid.uuid4() self.run_args = [] + self.runpaths.set_ert_case(self.sim_fs.name) paths = self.runpaths.get_paths( list(range(len(self.initial_mask))), self.iteration ) diff --git a/src/ert/run_models/base_run_model.py b/src/ert/run_models/base_run_model.py index fc2ca87f997..2fc53698ba8 100644 --- a/src/ert/run_models/base_run_model.py +++ b/src/ert/run_models/base_run_model.py @@ -132,8 +132,12 @@ def __init__( jobname_format=config.model_config.jobname_format_string, runpath_format=config.model_config.runpath_format_string, filename=str(config.runpath_file), - substitute=self.substitution_list.substitute_real_iter, + substitution_list=self.substitution_list, ) + if hasattr(self.simulation_arguments, "current_case"): + current_case = self.simulation_arguments.current_case + if current_case is not None: + self.run_paths.set_ert_case(current_case) self._send_event_callback: Optional[Callable[[object], None]] = None def add_send_event_callback(self, func: Callable[[object], None]) -> None: @@ -489,7 +493,7 @@ def _evaluate_and_postprocess( phase_string = f"Running simulation for iteration: {iteration}" self.setPhase(iteration, phase_string, indeterminate=False) - create_run_path(run_context, self.substitution_list, self.ert_config) + create_run_path(run_context, self.ert_config) phase_string = f"Pre processing for iteration: {iteration}" self.setPhaseName(phase_string, indeterminate=True) diff --git a/src/ert/runpaths.py b/src/ert/runpaths.py index f86af77acfc..cd2e9d9084b 100644 --- a/src/ert/runpaths.py +++ b/src/ert/runpaths.py @@ -1,5 +1,7 @@ from pathlib import Path -from typing import Callable, List, Union +from typing import List, Optional, Union + +from ert.substitution_list import SubstitutionList class Runpaths: @@ -31,22 +33,30 @@ def __init__( jobname_format: str, runpath_format: str, filename: Union[str, Path] = ".ert_runpath_list", - substitute: Callable[[str, int, int], str] = lambda x, *_: x, + substitution_list: Optional[SubstitutionList] = None, ): self._jobname_format = jobname_format self.runpath_list_filename = Path(filename) self._runpath_format = str(Path(runpath_format).resolve()) - self._substitute = substitute + self._substitution_list = substitution_list or SubstitutionList() + + def set_ert_case(self, case_name: str) -> None: + self._substitution_list[""] = case_name + self._substitution_list[""] = case_name def get_paths(self, realizations: List[int], iteration: int) -> List[str]: return [ - self._substitute(self._runpath_format, realization, iteration) + self._substitution_list.substitute_real_iter( + self._runpath_format, realization, iteration + ) for realization in realizations ] def get_jobnames(self, realizations: List[int], iteration: int) -> List[str]: return [ - self._substitute(self._jobname_format, realization, iteration) + self._substitution_list.substitute_real_iter( + self._jobname_format, realization, iteration + ) for realization in realizations ] @@ -78,10 +88,10 @@ def write_runpath_list( with open(self.runpath_list_filename, "w", encoding="utf-8") as filehandle: for iteration in iteration_numbers: for realization in realization_numbers: - job_name = self._substitute( + job_name = self._substitution_list.substitute_real_iter( self._jobname_format, realization, iteration ) - runpath = self._substitute( + runpath = self._substitution_list.substitute_real_iter( self._runpath_format, realization, iteration ) filehandle.write( diff --git a/src/ert/shared/hook_implementations/workflows/export_runpath.py b/src/ert/shared/hook_implementations/workflows/export_runpath.py index 21448c651f6..9a542062b02 100644 --- a/src/ert/shared/hook_implementations/workflows/export_runpath.py +++ b/src/ert/shared/hook_implementations/workflows/export_runpath.py @@ -35,7 +35,7 @@ def run(self, *args: str) -> None: jobname_format=config.model_config.jobname_format_string, runpath_format=config.model_config.runpath_format_string, filename=str(config.runpath_file), - substitute=config.substitution_list.substitute_real_iter, + substitution_list=config.substitution_list, ) run_paths.write_runpath_list(*self.get_ranges(_args)) diff --git a/src/ert/simulator/simulation_context.py b/src/ert/simulator/simulation_context.py index 5e4b3d873ec..ed8e8fad55c 100644 --- a/src/ert/simulator/simulation_context.py +++ b/src/ert/simulator/simulation_context.py @@ -112,13 +112,13 @@ def __init__( jobname_format=ert.ert_config.model_config.jobname_format_string, runpath_format=ert.ert_config.model_config.runpath_format_string, filename=str(ert.ert_config.runpath_file), - substitute=global_substitutions.substitute_real_iter, + substitution_list=global_substitutions, ), initial_mask=mask, iteration=itr, ) - create_run_path(self._run_context, global_substitutions, self._ert.ert_config) + create_run_path(self._run_context, self._ert.ert_config) self._ert.runWorkflows( HookRuntime.PRE_SIMULATION, None, self._run_context.sim_fs ) diff --git a/tests/unit_tests/config/test_defines.py b/tests/unit_tests/config/test_defines.py index cf505b3f1db..ef90c21f675 100644 --- a/tests/unit_tests/config/test_defines.py +++ b/tests/unit_tests/config/test_defines.py @@ -20,7 +20,7 @@ def read_jobname(config_file): runpath_format=ert_config.model_config.runpath_format_string, runpath_file="name", ) - create_run_path(run_context, ert_config.substitution_list, ert_config) + create_run_path(run_context, ert_config) return run_context[0].job_name diff --git a/tests/unit_tests/config/test_gen_kw_config.py b/tests/unit_tests/config/test_gen_kw_config.py index ce81d151fd9..71ffdb06e16 100644 --- a/tests/unit_tests/config/test_gen_kw_config.py +++ b/tests/unit_tests/config/test_gen_kw_config.py @@ -228,7 +228,7 @@ def test_gen_kw_is_log_or_not( "name", ) sample_prior(prior_ensemble, [0]) - create_run_path(prior, ert_config.substitution_list, ert_config) + create_run_path(prior, ert_config) assert re.match( parameters_regex, Path("simulations/realization-0/iter-0/parameters.txt").read_text( diff --git a/tests/unit_tests/run_models/test_base_run_model.py b/tests/unit_tests/run_models/test_base_run_model.py index d89653c2268..c831a2510ae 100644 --- a/tests/unit_tests/run_models/test_base_run_model.py +++ b/tests/unit_tests/run_models/test_base_run_model.py @@ -129,7 +129,8 @@ def test_check_if_runpath_exists( @pytest.mark.usefixtures("use_tmpdir") @pytest.mark.parametrize( - "run_path_format", ["realization-/iter-", "realization-"] + "run_path_format", + ["/realization-/iter-", "/realization-"], ) @pytest.mark.parametrize( "active_realizations", [[True], [True, True], [True, False], [False], [False, True]] @@ -138,7 +139,7 @@ def test_delete_run_path(run_path_format, active_realizations): simulation_arguments = EnsembleExperimentRunArguments( random_seed=None, active_realizations=active_realizations, - current_case=None, + current_case="Case_Name", target_case=None, start_iteration=0, iter_num=0, @@ -151,7 +152,9 @@ def test_delete_run_path(run_path_format, active_realizations): expected_removed = [] for iens, mask in enumerate(active_realizations): run_path = Path( - run_path_format.replace("", str(iens)).replace("", "0") + run_path_format.replace("", str(iens)) + .replace("", "0") + .replace("", "Case_Name") ) os.makedirs(run_path) assert run_path.exists() @@ -163,11 +166,15 @@ def test_delete_run_path(run_path_format, active_realizations): os.makedirs(share_path) model_config = ModelConfig(runpath_format_string=run_path_format) subs_list = SubstitutionList() + storage = MagicMock() + ensemble = MagicMock() + ensemble.ensemble_size = 1 + storage.get_ensemble_by_name.return_value = ensemble config = MagicMock() config.model_config = model_config config.substitution_list = subs_list - brm = BaseRunModel(simulation_arguments, config, None, None, None) + brm = BaseRunModel(simulation_arguments, config, storage, None, None) brm.rm_run_path() assert not any(path.exists() for path in expected_removed) assert all(path.parent.exists() for path in expected_removed) diff --git a/tests/unit_tests/storage/create_runpath.py b/tests/unit_tests/storage/create_runpath.py index e11158bcaca..658539c9c1d 100644 --- a/tests/unit_tests/storage/create_runpath.py +++ b/tests/unit_tests/storage/create_runpath.py @@ -43,7 +43,7 @@ def create_runpath( [i for i, active in enumerate(active_mask) if active], random_seed=random_seed, ) - create_run_path(prior, ert_config.substitution_list, ert_config) + create_run_path(prior, ert_config) return ert_config.ensemble_config, ensemble diff --git a/tests/unit_tests/test_enkf_main.py b/tests/unit_tests/test_enkf_main.py index 88dba8562db..f0f4a94aafc 100644 --- a/tests/unit_tests/test_enkf_main.py +++ b/tests/unit_tests/test_enkf_main.py @@ -97,7 +97,7 @@ def test_assert_symlink_deleted(snake_oil_field_example, storage): ) config = snake_oil_field_example sample_prior(prior_ensemble, range(prior_ensemble.ensemble_size)) - create_run_path(run_context, config.substitution_list, config) + create_run_path(run_context, config) # replace field file with symlink linkpath = f"{run_context[0].runpath}/permx.grdecl" @@ -108,7 +108,7 @@ def test_assert_symlink_deleted(snake_oil_field_example, storage): os.symlink(targetpath, linkpath) # recreate directory structure - create_run_path(run_context, config.substitution_list, config) + create_run_path(run_context, config) # ensure field symlink is replaced by file assert not os.path.islink(linkpath) diff --git a/tests/unit_tests/test_enkf_runpath.py b/tests/unit_tests/test_enkf_runpath.py index 4124648e188..315416dbaf0 100755 --- a/tests/unit_tests/test_enkf_runpath.py +++ b/tests/unit_tests/test_enkf_runpath.py @@ -27,7 +27,7 @@ def test_with_gen_kw(storage): "name", ) sample_prior(prior_ensemble, [0]) - create_run_path(prior, ert_config.substitution_list, ert_config) + create_run_path(prior, ert_config) assert os.path.exists( "storage/snake_oil/runpath/realization-0/iter-0/parameters.txt" ) @@ -54,7 +54,7 @@ def test_without_gen_kw(prior_ensemble): "name", ) sample_prior(prior_ensemble, [0]) - create_run_path(prior, ert_config.substitution_list, ert_config) + create_run_path(prior, ert_config) assert os.path.exists("storage/snake_oil/runpath/realization-0/iter-0") assert not os.path.exists( "storage/snake_oil/runpath/realization-0/iter-0/parameters.txt" @@ -82,9 +82,9 @@ def test_jobs_file_is_backed_up(storage): "name", ) sample_prior(prior_ensemble, [0]) - create_run_path(prior, ert_config.substitution_list, ert_config) + create_run_path(prior, ert_config) assert os.path.exists("storage/snake_oil/runpath/realization-0/iter-0/jobs.json") - create_run_path(prior, ert_config.substitution_list, ert_config) + create_run_path(prior, ert_config) iter0_output_files = os.listdir("storage/snake_oil/runpath/realization-0/iter-0/") jobs_files = [f for f in iter0_output_files if f.startswith("jobs.json")] assert len(jobs_files) > 1, "No backup created for jobs.json" diff --git a/tests/unit_tests/test_load_forward_model.py b/tests/unit_tests/test_load_forward_model.py index d3dae48f1d3..b6a0417f891 100644 --- a/tests/unit_tests/test_load_forward_model.py +++ b/tests/unit_tests/test_load_forward_model.py @@ -36,7 +36,7 @@ def func(config_text): ert_config.model_config.runpath_format_string, "name", ) - create_run_path(run_context, ert_config.substitution_list, ert_config) + create_run_path(run_context, ert_config) return ert_config, prior_ensemble yield func @@ -143,7 +143,7 @@ def test_load_forward_model_summary(summary_configuration, storage, expected, ca ert_config.model_config.runpath_format_string, "name", ) - create_run_path(run_context, ert_config.substitution_list, ert_config) + create_run_path(run_context, ert_config) facade = LibresFacade(ert_config) with caplog.at_level(logging.ERROR): loaded = facade.load_from_forward_model(prior_ensemble, [True], 0) @@ -258,7 +258,7 @@ def test_loading_gen_data_without_restart(storage): ert_config.model_config.runpath_format_string, "name", ) - create_run_path(run_context, ert_config.substitution_list, ert_config) + create_run_path(run_context, ert_config) run_path = Path("simulations/realization-0/iter-0/") with open(run_path / "response.out", "w", encoding="utf-8") as fout: fout.write("\n".join(["1", "2", "3"])) diff --git a/tests/unit_tests/test_run_path_creation.py b/tests/unit_tests/test_run_path_creation.py index 8720ad89612..40c75642886 100644 --- a/tests/unit_tests/test_run_path_creation.py +++ b/tests/unit_tests/test_run_path_creation.py @@ -44,7 +44,7 @@ def test_that_run_template_replace_symlink_does_not_write_to_source(prior_ensemb "I dont want to replace in this file", encoding="utf-8" ) os.symlink("start.txt", run_path / "result.txt") - create_run_path(run_context, ert_config.substitution_list, ert_config) + create_run_path(run_context, ert_config) assert (run_path / "result.txt").read_text( encoding="utf-8" ) == "I want to replace: 0" @@ -76,7 +76,7 @@ def test_run_template_replace_in_file_with_custom_define(prior_ensemble): run_context = ensemble_context( prior_ensemble, [True], 0, None, "", "name_%", "name" ) - create_run_path(run_context, ert_config.substitution_list, ert_config) + create_run_path(run_context, ert_config) assert ( Path(run_context[0].runpath) / "result.txt" ).read_text() == "I WANT TO REPLACE:my_custom_variable" @@ -111,9 +111,9 @@ def test_run_template_replace_in_file(key, expected, prior_ensemble): ert_config = ErtConfig.from_file("config.ert") run_context = ensemble_context( - prior_ensemble, [True], 0, None, "", "name_%", "name" + prior_ensemble, [True], 0, ert_config.substitution_list, "", "name_%", "name" ) - create_run_path(run_context, ert_config.substitution_list, ert_config) + create_run_path(run_context, ert_config) assert (Path(run_context[0].runpath) / "result.txt").read_text( encoding="utf-8" ) == f"I WANT TO REPLACE:{expected}" @@ -146,7 +146,7 @@ def test_run_template_replace_in_ecl(ecl_base, expected_file, prior_ensemble): run_context = ensemble_context( prior_ensemble, [True], 0, None, "", "name_%", "name" ) - create_run_path(run_context, ert_config.substitution_list, ert_config) + create_run_path(run_context, ert_config) assert ( Path(run_context[0].runpath) / expected_file ).read_text() == "I WANT TO REPLACE:1" @@ -186,9 +186,9 @@ def test_run_template_replace_in_ecl_data_file(key, expected, prior_ensemble): ert_config = ErtConfig.from_file("config.ert") run_context = ensemble_context( - prior_ensemble, [True], 0, None, "", "name_%", "name" + prior_ensemble, [True], 0, ert_config.substitution_list, "", "name_%", "name" ) - create_run_path(run_context, ert_config.substitution_list, ert_config) + create_run_path(run_context, ert_config) assert (Path(run_context[0].runpath) / "ECL_CASE0.DATA").read_text( encoding="utf-8" ) == f"I WANT TO REPLACE:{expected}" @@ -219,7 +219,7 @@ def test_that_error_is_raised_when_data_file_is_badly_encoded(prior_ensemble): ValueError, match="Unsupported non UTF-8 character found in file: .*MY_DATA_FILE.DATA", ): - create_run_path(run_context, ert_config.substitution_list, ert_config) + create_run_path(run_context, ert_config) @pytest.mark.usefixtures("use_tmpdir") @@ -245,7 +245,7 @@ def test_run_template_replace_in_file_name(prior_ensemble): run_context = ensemble_context( prior_ensemble, [True], 0, None, "", "name_%", "name" ) - create_run_path(run_context, ert_config.substitution_list, ert_config) + create_run_path(run_context, ert_config) assert ( Path(run_context[0].runpath) / "result.txt" ).read_text() == "Not important, name of the file is important" @@ -352,7 +352,7 @@ def test_that_runpath_substitution_remain_valid(prior_ensemble): ert_config.model_config.runpath_format_string, "name", ) - create_run_path(run_context, ert_config.substitution_list, ert_config) + create_run_path(run_context, ert_config) for i, realization in enumerate(run_context): assert str(Path().absolute()) + "/realization-" + str(i) + "/iter-0" in Path( @@ -388,14 +388,14 @@ def test_write_snakeoil_runpath_file(snake_oil_case, storage, itr): jobname_fmt, runpath_fmt, "a_file_name", - global_substitutions.substitute_real_iter, + global_substitutions, ), initial_mask=mask, iteration=itr, ) sample_prior(prior_ensemble, [i for i, active in enumerate(mask) if active]) - create_run_path(run_context, global_substitutions, ert_config) + create_run_path(run_context, ert_config) for i, _ in enumerate(run_context): if not mask[i]: @@ -448,7 +448,7 @@ def test_assert_export(prior_ensemble): substitution_list=ert_config.substitution_list, ) sample_prior(prior_ensemble, [0]) - create_run_path(run_context, ert_config.substitution_list, ert_config) + create_run_path(run_context, ert_config) assert runpath_list_file.exists() assert runpath_list_file.name == "test_runpath_list.txt" @@ -475,7 +475,7 @@ def _create_runpath(ert_config: ErtConfig, storage: StorageAccessor) -> RunConte ert_config.model_config.runpath_format_string, ert_config.runpath_file, ) - create_run_path(run_context, ert_config.substitution_list, ert_config) + create_run_path(run_context, ert_config) return run_context @@ -558,3 +558,36 @@ def test_that_runpaths_are_raised_when_invalid(run_path, expected_raise, msg): _ = ErtConfig.from_file("config.ert") else: _ = ErtConfig.from_file("config.ert") + + +@pytest.mark.usefixtures("use_tmpdir") +@pytest.mark.parametrize( + "placeholder", + ["", ""], +) +def test_assert_ertcase_replaced_in_runpath(placeholder, prior_ensemble, storage): + # Write a minimal config file with env + with open("config_file.ert", "w", encoding="utf-8") as fout: + fout.write( + dedent( + f""" + NUM_REALIZATIONS 1 + JOBNAME a_name_%d + RUNPATH simulations/{placeholder}/realization-/iter- + """ + ) + ) + ert_config = ErtConfig.from_file("config_file.ert") + _create_runpath(ert_config, storage) + + runpath_file = ( + f"{os.getcwd()}/simulations/{prior_ensemble.name}/realization-0/iter-0" + ) + + assert ( + ert_config.runpath_file.read_text("utf-8") + == f"000 {runpath_file} a_name_0 000\n" + ) + assert Path(runpath_file).exists() + jobs_json = Path(runpath_file) / "jobs.json" + assert jobs_json.exists() diff --git a/tests/unit_tests/test_runpaths.py b/tests/unit_tests/test_runpaths.py index 891d8d3438d..86e6a25280e 100644 --- a/tests/unit_tests/test_runpaths.py +++ b/tests/unit_tests/test_runpaths.py @@ -58,7 +58,7 @@ def test_runpath_file(tmp_path, job_format, runpath_format, expected_contents): job_format, runpath_format, runpath_file, - context.substitute_real_iter, + context, ) runpaths.write_runpath_list([0, 1], [3, 4]) @@ -74,7 +74,7 @@ def test_runpath_file_writer_substitution(tmp_path): "_job", "/path//ensemble-/iteration", runpath_file, - context.substitute_real_iter, + context, ) runpaths.write_runpath_list([1], [1]) diff --git a/tests/unit_tests/test_summary_response.py b/tests/unit_tests/test_summary_response.py index 0419e725bd4..5e2895cf767 100644 --- a/tests/unit_tests/test_summary_response.py +++ b/tests/unit_tests/test_summary_response.py @@ -48,7 +48,7 @@ def test_load_summary_response_restart_not_zero(tmpdir, snapshot, request, stora "name", ) - create_run_path(prior, ert_config.substitution_list, ert_config) + create_run_path(prior, ert_config) os.chdir(sim_path) shutil.copy(test_path / "PRED_RUN.SMSPEC", "PRED_RUN.SMSPEC") shutil.copy(test_path / "PRED_RUN.UNSMRY", "PRED_RUN.UNSMRY")