Skip to content

Commit

Permalink
Replace magic string <ERTCASE> in the runpath
Browse files Browse the repository at this point in the history
  • Loading branch information
DanSava committed Feb 27, 2024
1 parent 759f431 commit 3e89cb1
Show file tree
Hide file tree
Showing 16 changed files with 99 additions and 49 deletions.
7 changes: 2 additions & 5 deletions src/ert/enkf_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,13 +197,10 @@ def sample_prior(

def create_run_path(
run_context: RunContext,
substitution_list: SubstitutionList,
ert_config: ErtConfig,
) -> None:
t = time.perf_counter()
substitution_list = copy(substitution_list)
substitution_list["<ERT-CASE>"] = run_context.sim_fs.name
substitution_list["<ERTCASE>"] = run_context.sim_fs.name
substitution_list = ert_config.substitution_list
for iens, run_arg in enumerate(run_context):
run_path = Path(run_arg.runpath)
if run_context.is_active(iens):
Expand Down Expand Up @@ -279,7 +276,7 @@ def ensemble_context(
jobname_format=jobname_format,
runpath_format=runpath_format,
filename=runpath_file,
substitute=substitution_list.substitute_real_iter,
substitution_list=substitution_list,
)
return RunContext(
sim_fs=case,
Expand Down
1 change: 1 addition & 0 deletions src/ert/run_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ class RunContext:
def __post_init__(self) -> None:
self.run_id = uuid.uuid4()
self.run_args = []
self.runpaths.set_ert_case(self.sim_fs.name)
paths = self.runpaths.get_paths(
list(range(len(self.initial_mask))), self.iteration
)
Expand Down
6 changes: 4 additions & 2 deletions src/ert/run_models/base_run_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,8 +132,10 @@ def __init__(
jobname_format=config.model_config.jobname_format_string,
runpath_format=config.model_config.runpath_format_string,
filename=str(config.runpath_file),
substitute=self.substitution_list.substitute_real_iter,
substitution_list=self.substitution_list,
)
if hasattr(self.simulation_arguments, "current_case"):
self.run_paths.set_ert_case(self.simulation_arguments.current_case)
self._send_event_callback: Optional[Callable[[object], None]] = None

def add_send_event_callback(self, func: Callable[[object], None]) -> None:
Expand Down Expand Up @@ -489,7 +491,7 @@ def _evaluate_and_postprocess(

phase_string = f"Running simulation for iteration: {iteration}"
self.setPhase(iteration, phase_string, indeterminate=False)
create_run_path(run_context, self.substitution_list, self.ert_config)
create_run_path(run_context, self.ert_config)

phase_string = f"Pre processing for iteration: {iteration}"
self.setPhaseName(phase_string, indeterminate=True)
Expand Down
22 changes: 16 additions & 6 deletions src/ert/runpaths.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
from pathlib import Path
from typing import Callable, List, Union

from ert.substitution_list import SubstitutionList


class Runpaths:
"""The Runpaths are the ensemble workspace directories.
Expand Down Expand Up @@ -31,22 +33,30 @@ def __init__(
jobname_format: str,
runpath_format: str,
filename: Union[str, Path] = ".ert_runpath_list",
substitute: Callable[[str, int, int], str] = lambda x, *_: x,
substitution_list: SubstitutionList = SubstitutionList(),
):
self._jobname_format = jobname_format
self.runpath_list_filename = Path(filename)
self._runpath_format = str(Path(runpath_format).resolve())
self._substitute = substitute
self._substitution_list = substitution_list

def set_ert_case(self, case_name: str) -> None:
self._substitution_list["<ERT-CASE>"] = case_name
self._substitution_list["<ERTCASE>"] = case_name

def get_paths(self, realizations: List[int], iteration: int) -> List[str]:
return [
self._substitute(self._runpath_format, realization, iteration)
self._substitution_list.substitute_real_iter(
self._runpath_format, realization, iteration
)
for realization in realizations
]

def get_jobnames(self, realizations: List[int], iteration: int) -> List[str]:
return [
self._substitute(self._jobname_format, realization, iteration)
self._substitution_list.substitute_real_iter(
self._jobname_format, realization, iteration
)
for realization in realizations
]

Expand Down Expand Up @@ -78,10 +88,10 @@ def write_runpath_list(
with open(self.runpath_list_filename, "w", encoding="utf-8") as filehandle:
for iteration in iteration_numbers:
for realization in realization_numbers:
job_name = self._substitute(
job_name = self._substitution_list.substitute_real_iter(
self._jobname_format, realization, iteration
)
runpath = self._substitute(
runpath = self._substitution_list.substitute_real_iter(
self._runpath_format, realization, iteration
)
filehandle.write(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def run(self, *args: str) -> None:
jobname_format=config.model_config.jobname_format_string,
runpath_format=config.model_config.runpath_format_string,
filename=str(config.runpath_file),
substitute=config.substitution_list.substitute_real_iter,
substitution_list=config.substitution_list,
)
run_paths.write_runpath_list(*self.get_ranges(_args))

Expand Down
4 changes: 2 additions & 2 deletions src/ert/simulator/simulation_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,13 +112,13 @@ def __init__(
jobname_format=ert.ert_config.model_config.jobname_format_string,
runpath_format=ert.ert_config.model_config.runpath_format_string,
filename=str(ert.ert_config.runpath_file),
substitute=global_substitutions.substitute_real_iter,
substitution_list=global_substitutions,
),
initial_mask=mask,
iteration=itr,
)

create_run_path(self._run_context, global_substitutions, self._ert.ert_config)
create_run_path(self._run_context, self._ert.ert_config)
self._ert.runWorkflows(
HookRuntime.PRE_SIMULATION, None, self._run_context.sim_fs
)
Expand Down
2 changes: 1 addition & 1 deletion tests/unit_tests/config/test_defines.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def read_jobname(config_file):
runpath_format=ert_config.model_config.runpath_format_string,
runpath_file="name",
)
create_run_path(run_context, ert_config.substitution_list, ert_config)
create_run_path(run_context, ert_config)
return run_context[0].job_name


Expand Down
2 changes: 1 addition & 1 deletion tests/unit_tests/config/test_gen_kw_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ def test_gen_kw_is_log_or_not(
"name",
)
sample_prior(prior_ensemble, [0])
create_run_path(prior, ert_config.substitution_list, ert_config)
create_run_path(prior, ert_config)
assert re.match(
parameters_regex,
Path("simulations/realization-0/iter-0/parameters.txt").read_text(
Expand Down
15 changes: 11 additions & 4 deletions tests/unit_tests/run_models/test_base_run_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,8 @@ def test_check_if_runpath_exists(

@pytest.mark.usefixtures("use_tmpdir")
@pytest.mark.parametrize(
"run_path_format", ["realization-<IENS>/iter-<ITER>", "realization-<IENS>"]
"run_path_format",
["<ERTCASE>/realization-<IENS>/iter-<ITER>", "<ERTCASE>/realization-<IENS>"],
)
@pytest.mark.parametrize(
"active_realizations", [[True], [True, True], [True, False], [False], [False, True]]
Expand All @@ -138,7 +139,7 @@ def test_delete_run_path(run_path_format, active_realizations):
simulation_arguments = EnsembleExperimentRunArguments(
random_seed=None,
active_realizations=active_realizations,
current_case=None,
current_case="Case_Name",
target_case=None,
start_iteration=0,
iter_num=0,
Expand All @@ -151,7 +152,9 @@ def test_delete_run_path(run_path_format, active_realizations):
expected_removed = []
for iens, mask in enumerate(active_realizations):
run_path = Path(
run_path_format.replace("<IENS>", str(iens)).replace("<ITER>", "0")
run_path_format.replace("<IENS>", str(iens))
.replace("<ITER>", "0")
.replace("<ERTCASE>", "Case_Name")
)
os.makedirs(run_path)
assert run_path.exists()
Expand All @@ -163,11 +166,15 @@ def test_delete_run_path(run_path_format, active_realizations):
os.makedirs(share_path)
model_config = ModelConfig(runpath_format_string=run_path_format)
subs_list = SubstitutionList()
storage = MagicMock()
ensemble = MagicMock()
ensemble.ensemble_size = 1
storage.get_ensemble_by_name.return_value = ensemble
config = MagicMock()
config.model_config = model_config
config.substitution_list = subs_list

brm = BaseRunModel(simulation_arguments, config, None, None, None)
brm = BaseRunModel(simulation_arguments, config, storage, None, None)
brm.rm_run_path()
assert not any(path.exists() for path in expected_removed)
assert all(path.parent.exists() for path in expected_removed)
Expand Down
2 changes: 1 addition & 1 deletion tests/unit_tests/storage/create_runpath.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def create_runpath(
[i for i, active in enumerate(active_mask) if active],
random_seed=random_seed,
)
create_run_path(prior, ert_config.substitution_list, ert_config)
create_run_path(prior, ert_config)
return ert_config.ensemble_config, ensemble


Expand Down
4 changes: 2 additions & 2 deletions tests/unit_tests/test_enkf_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ def test_assert_symlink_deleted(snake_oil_field_example, storage):
)
config = snake_oil_field_example
sample_prior(prior_ensemble, range(prior_ensemble.ensemble_size))
create_run_path(run_context, config.substitution_list, config)
create_run_path(run_context, config)

# replace field file with symlink
linkpath = f"{run_context[0].runpath}/permx.grdecl"
Expand All @@ -108,7 +108,7 @@ def test_assert_symlink_deleted(snake_oil_field_example, storage):
os.symlink(targetpath, linkpath)

# recreate directory structure
create_run_path(run_context, config.substitution_list, config)
create_run_path(run_context, config)

# ensure field symlink is replaced by file
assert not os.path.islink(linkpath)
Expand Down
8 changes: 4 additions & 4 deletions tests/unit_tests/test_enkf_runpath.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ def test_with_gen_kw(storage):
"name",
)
sample_prior(prior_ensemble, [0])
create_run_path(prior, ert_config.substitution_list, ert_config)
create_run_path(prior, ert_config)
assert os.path.exists(
"storage/snake_oil/runpath/realization-0/iter-0/parameters.txt"
)
Expand All @@ -54,7 +54,7 @@ def test_without_gen_kw(prior_ensemble):
"name",
)
sample_prior(prior_ensemble, [0])
create_run_path(prior, ert_config.substitution_list, ert_config)
create_run_path(prior, ert_config)
assert os.path.exists("storage/snake_oil/runpath/realization-0/iter-0")
assert not os.path.exists(
"storage/snake_oil/runpath/realization-0/iter-0/parameters.txt"
Expand Down Expand Up @@ -82,9 +82,9 @@ def test_jobs_file_is_backed_up(storage):
"name",
)
sample_prior(prior_ensemble, [0])
create_run_path(prior, ert_config.substitution_list, ert_config)
create_run_path(prior, ert_config)
assert os.path.exists("storage/snake_oil/runpath/realization-0/iter-0/jobs.json")
create_run_path(prior, ert_config.substitution_list, ert_config)
create_run_path(prior, ert_config)
iter0_output_files = os.listdir("storage/snake_oil/runpath/realization-0/iter-0/")
jobs_files = [f for f in iter0_output_files if f.startswith("jobs.json")]
assert len(jobs_files) > 1, "No backup created for jobs.json"
6 changes: 3 additions & 3 deletions tests/unit_tests/test_load_forward_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def func(config_text):
ert_config.model_config.runpath_format_string,
"name",
)
create_run_path(run_context, ert_config.substitution_list, ert_config)
create_run_path(run_context, ert_config)
return ert_config, prior_ensemble

yield func
Expand Down Expand Up @@ -143,7 +143,7 @@ def test_load_forward_model_summary(summary_configuration, storage, expected, ca
ert_config.model_config.runpath_format_string,
"name",
)
create_run_path(run_context, ert_config.substitution_list, ert_config)
create_run_path(run_context, ert_config)
facade = LibresFacade(ert_config)
with caplog.at_level(logging.ERROR):
loaded = facade.load_from_forward_model(prior_ensemble, [True], 0)
Expand Down Expand Up @@ -258,7 +258,7 @@ def test_loading_gen_data_without_restart(storage):
ert_config.model_config.runpath_format_string,
"name",
)
create_run_path(run_context, ert_config.substitution_list, ert_config)
create_run_path(run_context, ert_config)
run_path = Path("simulations/realization-0/iter-0/")
with open(run_path / "response.out", "w", encoding="utf-8") as fout:
fout.write("\n".join(["1", "2", "3"]))
Expand Down
Loading

0 comments on commit 3e89cb1

Please sign in to comment.