From 5d1efd841ff1925ac0c66fa3d006f70a5e24f76d Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Thu, 25 Jul 2024 09:19:15 +0200 Subject: [PATCH 1/2] ensure that stdout/stderr strings are utf8 --- src/radical/pilot/agent/staging_output/default.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/radical/pilot/agent/staging_output/default.py b/src/radical/pilot/agent/staging_output/default.py index b299b8ff3..c75fc1737 100644 --- a/src/radical/pilot/agent/staging_output/default.py +++ b/src/radical/pilot/agent/staging_output/default.py @@ -163,7 +163,8 @@ def _handle_task_stdio(self, task): # TODO: disable this at scale? if task.get('stderr_file') and os.path.isfile(task['stderr_file']): - with ru.ru_open(task['stderr_file'], 'r') as stderr_f: + with ru.ru_open(task['stderr_file'], 'r', errors='ignore') \ + as stderr_f: try: txt = ru.as_string(stderr_f.read()) except UnicodeDecodeError: @@ -173,7 +174,8 @@ def _handle_task_stdio(self, task): # to help with ID mapping, also parse for PRTE output: # [batch3:122527] JOB [3673,4] EXECUTING - with ru.ru_open(task['stderr_file'], 'r') as stderr_f: + with ru.ru_open(task['stderr_file'], 'r', errors='ignore') \ + as stderr_f: for line in stderr_f.readlines(): line = line.strip() @@ -191,7 +193,7 @@ def _handle_task_stdio(self, task): if os.path.isfile(task_prof): pids = {} try: - with ru.ru_open(task_prof, 'r') as prof_f: + with ru.ru_open(task_prof, 'r', errors='ignore') as prof_f: txt = ru.as_string(prof_f.read()) for line in txt.split("\n"): if not line: From 2f8d15c94160657d80ab3a531b349eaec11bf082 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 26 Jul 2024 14:55:11 +0200 Subject: [PATCH 2/2] remove obsolete code, also handle stdout --- .../pilot/agent/staging_output/default.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/radical/pilot/agent/staging_output/default.py b/src/radical/pilot/agent/staging_output/default.py index c75fc1737..6931d54b9 100644 --- a/src/radical/pilot/agent/staging_output/default.py +++ b/src/radical/pilot/agent/staging_output/default.py @@ -150,7 +150,8 @@ def _handle_task_stdio(self, task): # TODO: disable this at scale? if task.get('stdout_file') and os.path.isfile(task['stdout_file']): - with ru.ru_open(task['stdout_file'], 'r') as stdout_f: + with ru.ru_open(task['stdout_file'], 'r', errors='ignore') \ + as stdout_f: try: txt = ru.as_string(stdout_f.read()) except UnicodeDecodeError: @@ -172,20 +173,6 @@ def _handle_task_stdio(self, task): task['stderr'] += rpu.tail(txt) - # to help with ID mapping, also parse for PRTE output: - # [batch3:122527] JOB [3673,4] EXECUTING - with ru.ru_open(task['stderr_file'], 'r', errors='ignore') \ - as stderr_f: - - for line in stderr_f.readlines(): - line = line.strip() - if not line: - continue - if line[0] == '[' and line.endswith('EXECUTING'): - elems = line.replace('[', '').replace(']', '').split() - tid = elems[2] - self._log.info('PRTE IDMAP: %s:%s' % (tid, uid)) - self._prof.prof('staging_stderr_stop', uid=uid) self._prof.prof('staging_uprof_start', uid=uid)