Improve exception handling around init/loading of enkf nodes

src/clib/lib/config/config_parser.cpp

@@ -1,4 +1,5 @@
 #include <filesystem>
+#include <fstream>
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -657,15 +658,33 @@ config_parse(config_parser_type *config, const char *filename,
         hash_iter_free(keys);
     }
 
-    if (util_file_readable(filename)) {
+    std::ifstream file_handler;
+    file_handler.exceptions(std::ifstream::failbit | std::ifstream::badbit);
+    bool file_readable_check_succeeded = true;
+    try {
+        file_handler.open(filename);
+    } catch (std::ios_base::failure &err) {
+        file_readable_check_succeeded = false;
+        auto error_message = fmt::format(
+            "could not open file `{}` for parsing - {}", filename, err.what());
+        content->parse_errors.push_back(error_message);
+    }
+    try {
+        file_handler.close();
+    } catch (std::ios_base::failure &err) {
+        file_readable_check_succeeded = false;
+        auto error_message =
+            fmt::format("unexpected failure to close `{}` while "
+                        "checking if readable for parsing",
+                        filename, err.what());
+        content->parse_errors.push_back(error_message);
+    }
+
+    if (file_readable_check_succeeded) {
         path_stack_type *path_stack = path_stack_alloc();
         config_parse__(config, content, path_stack, filename, comment_string,
                        include_kw, define_kw, unrecognized_behaviour, validate);
         path_stack_free(path_stack);
-    } else {
-        std::string error_message =
-            util_alloc_sprintf("Could not open file:%s for parsing", filename);
-        content->parse_errors.push_back(error_message);
     }
 
     if (content->parse_errors.size() == 0)

src/clib/lib/enkf/enkf_node.cpp

@@ -257,8 +257,22 @@ bool enkf_node_store(enkf_node_type *enkf_node, enkf_fs_type *fs,
 bool enkf_node_try_load(enkf_node_type *enkf_node, enkf_fs_type *fs,
                         node_id_type node_id) {
     if (enkf_node_has_data(enkf_node, fs, node_id)) {
-        enkf_node_load(enkf_node, fs, node_id);
-        return true;
+        try {
+            enkf_node_load(enkf_node, fs, node_id);
+            return true;
+        } catch (std::range_error &err) {
+            std::string error_message = fmt::format(
+                "failed to load node `{}` of type `{}` - {}",
+                enkf_node->node_key,
+                enkf_types_get_impl_name(enkf_node->config->impl_type),
+                err.what());
+            throw std::range_error(error_message);
+        } catch (...) {
+            std::string context_message = fmt::format(
+                "failed to load node `{}` of type `{}`", enkf_node->node_key,
+                enkf_types_get_impl_name(enkf_node->config->impl_type));
+            std::throw_with_nested(std::runtime_error(context_message));
+        }
     } else
         return false;
 }
@@ -419,11 +433,21 @@ void enkf_node_deserialize(enkf_node_type *enkf_node, enkf_fs_type *fs,
 */
 bool enkf_node_initialize(enkf_node_type *enkf_node, int iens) {
     if (enkf_node->initialize != NULL) {
-        char *init_file =
-            enkf_config_node_alloc_initfile(enkf_node->config, NULL, iens);
-        bool init = enkf_node->initialize(enkf_node->data, iens, init_file);
-        free(init_file);
-        return init;
+        std::unique_ptr<char> init_file(
+            enkf_config_node_alloc_initfile(enkf_node->config, NULL, iens));
+        try {
+            bool init =
+                enkf_node->initialize(enkf_node->data, iens, init_file.get());
+            return init;
+        } catch (std::exception &err) {
+            std::string err_msg = fmt::format(
+                "failed to initialize node `{}` of type `{}` for realization "
+                "number {} - {}",
+                enkf_node->node_key,
+                enkf_types_get_impl_name(enkf_node->config->impl_type), iens,
+                err.what());
+            throw std::runtime_error(err_msg);
+        }
     } else
         return false; /* No init performed */
 }

src/clib/lib/enkf/field.cpp

@@ -1,4 +1,5 @@
 #include <filesystem>
+#include <fstream>
 
 #include <Eigen/Dense>
 #include <cmath>
@@ -825,16 +826,34 @@ bool field_fload_typed(field_type *field, const char *filename,
 
 static bool field_fload_custom__(field_type *field, const char *filename,
                                  bool keep_inactive) {
-    if (util_file_readable(filename)) {
-        field_file_format_type file_type =
-            field_config_guess_file_type(filename);
-        if (file_type == UNDEFINED_FORMAT)
-            util_abort("%s - could not automagically infer type for file: %s\n",
-                       __func__, filename);
+    std::ifstream file_handler;
+    file_handler.exceptions(std::ifstream::failbit | std::ifstream::badbit);
+    try {
+        file_handler.open(filename);
+    } catch (std::ios_base::failure &err) {
+        auto error_meessage =
+            fmt::format("failed to open `{}` - {}", filename, err.what());
+        throw std::runtime_error(error_meessage);
+    }
+    try {
+        file_handler.close();
+    } catch (std::ios_base::failure &err) {
+        auto error_meessage =
+            fmt::format("unexpected failure to close `{}` while checkinf if "
+                        "file readable - {}",
+                        filename, err.what());
+        throw std::runtime_error(error_meessage);
+    }
 
-        return field_fload_typed(field, filename, file_type, keep_inactive);
-    } else
-        return false;
+    field_file_format_type file_type = field_config_guess_file_type(filename);
+    if (file_type == UNDEFINED_FORMAT) {
+        std::string error_meessage = fmt::format("could not infer type "
+                                                 "for file: %s\n",
+                                                 filename);
+        throw std::runtime_error(error_meessage);
+    }
+
+    return field_fload_typed(field, filename, file_type, keep_inactive);
 }
 
 bool field_fload(field_type *field, const char *filename) {

src/clib/lib/enkf/gen_kw_config.cpp

@@ -1,4 +1,5 @@
 #include <filesystem>
+#include <numeric>
 
 #include <stdlib.h>
 #include <string.h>
@@ -8,6 +9,7 @@
 #include <ert/util/vector.h>
 
 #include <ert/config/config_parser.hpp>
+#include <ert/logging.hpp>
 
 #include <ert/enkf/config_keys.hpp>
 #include <ert/enkf/enkf_defaults.hpp>
@@ -17,6 +19,7 @@
 #include <ert/enkf/trans_func.hpp>
 
 namespace fs = std::filesystem;
+static auto logger = ert::get_logger("gen_kw_config");
 
 typedef struct {
     char *name;
@@ -102,6 +105,18 @@ void gen_kw_config_set_parameter_file(gen_kw_config_type *config,
         config_content_type *content =
             config_parse(parser, parameter_file, "--", NULL, NULL, NULL,
                          CONFIG_UNRECOGNIZED_ADD, false);
+        if (!content->valid) {
+            auto header = fmt::format(
+                "encountered errors while parsing GEN_KW parameter file {}",
+                parameter_file);
+            auto errors =
+                std::reduce(content->parse_errors.begin(),
+                            content->parse_errors.end(), std::string("\n"));
+            logger->warning("{}\n{}", header, errors);
+        }
+        for (auto parse_error : content->parse_errors) {
+            logger->warning(parse_error);
+        }
         for (int item_index = 0; item_index < config_content_get_size(content);
              item_index++) {
             const config_content_node_type *node =

src/ert/_c_wrappers/enkf/data/enkf_node.py

@@ -107,10 +107,10 @@ def tryLoad(self, fs: EnkfFs, node_id: NodeId) -> bool:
     def name(self) -> str:
         return self._get_name()
 
-    def load(self, fs: EnkfFs, node_id: NodeId):
+    def load(self, fs: EnkfFs, node_id: NodeId) -> None:
         if not self.tryLoad(fs, node_id):
-            raise Exception(
-                f"Could not load node: {self.name()} iens: {node_id.iens} "
+            raise RuntimeError(
+                f"Could not load node: {self.name()!r}, iens: {node_id.iens} "
                 f"report: {node_id.report_step}"
             )
 

src/ert/_c_wrappers/enkf/enkf_obs.py

@@ -1,3 +1,4 @@
+import os
 from typing import Iterator, List, Optional, Union
 
 from cwrap import BaseCClass
@@ -116,6 +117,11 @@ def free(self):
         self._free()
 
     def load(self, config_file, std_cutoff):
+        if not os.access(config_file, os.R_OK):
+            raise RuntimeError(
+                "Do not have permission to open observation "
+                f"config file {config_file!r}"
+            )
         _clib.enkf_obs.load(self, config_file, std_cutoff)
 
     @property

src/ert/ensemble_evaluator/_wait_for_evaluator.py

@@ -8,6 +8,8 @@
 
 logger = logging.getLogger(__name__)
 
+WAIT_FOR_EVALUATOR_TIMEOUT = 60
+
 
 def get_ssl_context(cert: Optional[Union[str, bytes]]) -> Optional[ssl.SSLContext]:
     if cert is None:
@@ -41,9 +43,11 @@ async def wait_for_evaluator(  # pylint: disable=too-many-arguments
     token: Optional[str] = None,
     cert: Optional[Union[str, bytes]] = None,
     healthcheck_endpoint: str = "/healthcheck",
-    timeout: float = 60,
+    timeout: Optional[float] = None,
     connection_timeout: float = 2,
 ) -> None:
+    if timeout is None:
+        timeout = WAIT_FOR_EVALUATOR_TIMEOUT
     healthcheck_url = base_url + healthcheck_endpoint
     start = time.time()
     sleep_time = 0.2

tests/unit_tests/cli/test_integration_cli.py

@@ -1,5 +1,6 @@
 import asyncio
 import fileinput
+import logging
 import os
 import shutil
 import threading
@@ -11,7 +12,7 @@
 import pytest
 
 import ert.shared
-from ert import LibresFacade
+from ert import LibresFacade, ensemble_evaluator
 from ert.__main__ import ert_parser
 from ert._c_wrappers.config.config_parser import ConfigValidationError
 from ert._c_wrappers.enkf import EnKFMain, ErtConfig
@@ -436,3 +437,103 @@ def test_that_prior_is_not_overwritten_in_ensemble_experiment(
                 pd.testing.assert_frame_equal(parameter_values, prior_values)
         else:
             pd.testing.assert_frame_equal(parameter_values, prior_values)
+
+
+def test_field_init_file_not_readable(copy_case, monkeypatch):
+    monkeypatch.setattr(
+        ensemble_evaluator._wait_for_evaluator, "WAIT_FOR_EVALUATOR_TIMEOUT", 5
+    )
+    copy_case("snake_oil_field")
+    config_file_name = "snake_oil_field.ert"
+    field_file_rel_path = "fields/permx0.grdecl"
+    os.chmod(field_file_rel_path, 0x0)
+
+    try:
+        run_ert_test_run(config_file_name)
+    except ErtCliError as err:
+        assert "failed to open" in str(err)
+
+
+def test_surface_init_fails_during_forward_model_callback(copy_case):
+    copy_case("snake_oil_field")
+    config_file_name = "snake_oil_surface.ert"
+    with open(config_file_name, mode="r+", encoding="utf-8") as config_file_handler:
+        content_lines = config_file_handler.read().splitlines()
+        index_line_with_surface_top = [
+            index
+            for index, line in enumerate(content_lines)
+            if line.startswith("SURFACE TOP")
+        ][0]
+        line_with_surface_top = content_lines[index_line_with_surface_top]
+        breaking_line_with_surface_top = line_with_surface_top + " FORWARD_INIT:True"
+        content_lines[index_line_with_surface_top] = breaking_line_with_surface_top
+        config_file_handler.seek(0)
+        config_file_handler.write("\n".join(content_lines))
+
+    try:
+        run_ert_test_run(config_file_name)
+    except ErtCliError as err:
+        assert "Failed to initialize node" in str(err)
+
+
+def test_config_parser_fails_gracefully_on_unreadable_config_file(copy_case, caplog):
+    """we cannot test on the config file directly, as the argument parser already check
+    if the file is readable. so we use the GEN_KW parameter file which is also parsed
+    using our config parser."""
+
+    copy_case("snake_oil_field")
+    config_file_name = "snake_oil_surface.ert"
+
+    with open(config_file_name, mode="r", encoding="utf-8") as config_file_handler:
+        content_lines = config_file_handler.read().splitlines()
+
+    index_line_with_gen_kw = [
+        index for index, line in enumerate(content_lines) if line.startswith("GEN_KW")
+    ][0]
+    gen_kw_parameter_file = content_lines[index_line_with_gen_kw].split(" ")[4]
+    os.chmod(gen_kw_parameter_file, 0x0)
+    gen_kw_parameter_file_abs_path = os.path.join(os.getcwd(), gen_kw_parameter_file)
+    caplog.set_level(logging.WARNING)
+
+    ErtConfig.from_file(config_file_name)
+
+    assert (
+        f"could not open file `{gen_kw_parameter_file_abs_path}` for parsing"
+        in caplog.text
+    )
+
+
+def test_unopenable_observation_config_fails_gracefully(copy_case):
+    copy_case("snake_oil_field")
+    config_file_name = "snake_oil_field.ert"
+    with open(config_file_name, mode="r", encoding="utf-8") as config_file_handler:
+        content_lines = config_file_handler.read().splitlines()
+    index_line_with_observation_config = [
+        index
+        for index, line in enumerate(content_lines)
+        if line.startswith("OBS_CONFIG")
+    ][0]
+    line_with_observation_config = content_lines[index_line_with_observation_config]
+    observation_config_rel_path = line_with_observation_config.split(" ")[1]
+    observation_config_abs_path = os.path.join(os.getcwd(), observation_config_rel_path)
+    os.chmod(observation_config_abs_path, 0x0)
+
+    try:
+        run_ert_test_run(config_file_name)
+    except RuntimeError as err:
+        assert (
+            "Do not have permission to open observation config file "
+            f"{observation_config_abs_path!r}" in str(err)
+        )
+
+
+def run_ert_test_run(config_file: str) -> None:
+    parser = ArgumentParser(prog="test_run")
+    parsed = ert_parser(
+        parser,
+        [
+            TEST_RUN_MODE,
+            config_file,
+        ],
+    )
+    run_cli(parsed)