Improve error handling in runner.py job.run(...)

This commit wraps the `job.run(...)` method in a try-except block making sure the `Exited` message is sent if the job_runner crashes. This fixes the bug where ert is hanging when unhandled exceptions are raised in the job runner.

src/_ert/forward_model_runner/cli.py

@@ -24,7 +24,7 @@ def _setup_reporters(
     ee_token=None,
     ee_cert_path=None,
     experiment_id=None,
-):
+) -> typing.List[reporting.Reporter]:
     reporters: typing.List[reporting.Reporter] = []
     if is_interactive_run:
         reporters.append(reporting.Interactive())

src/_ert/forward_model_runner/job.py

@@ -83,6 +83,13 @@ def __init__(self, job_data, index, sleep_interval=1):
         self.std_out = job_data.get("stdout")
 
     def run(self):
+        try:
+            for msg in self._run():
+                yield msg
+        except Exception as e:
+            yield Exited(self, exit_code=1).with_error(str(e))
+
+    def _run(self):
         start_message = Start(self)
 
         errors = self._check_job_files()

src/_ert/forward_model_runner/runner.py

@@ -2,6 +2,7 @@
 import json
 import os
 from pathlib import Path
+from typing import List
 
 from _ert.forward_model_runner.job import Job
 from _ert.forward_model_runner.reporting.message import Checksum, Finish, Init
@@ -21,7 +22,7 @@ def __init__(self, jobs_data):
         if self.simulation_id is not None:
             os.environ["ERT_RUN_ID"] = self.simulation_id
 
-        self.jobs = []
+        self.jobs: List[Job] = []
         for index, job_data in enumerate(job_data_list):
             self.jobs.append(Job(job_data, index))