diff --git a/lib/gpt/core/log.py b/lib/gpt/core/log.py index 397c7e69..8cbb2110 100644 --- a/lib/gpt/core/log.py +++ b/lib/gpt/core/log.py @@ -16,12 +16,16 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # -import gpt, sys, os +import gpt, sys, os, signal, datetime from inspect import getframeinfo, stack verbose = gpt.default.is_verbose("message_context") +last_context = "" + def message(*a, force_output=False): + global last_context + # conversion to string can be an mpi process (i.e. for lattice), # so need to do it on all ranks s = " ".join([str(x) for x in a]) @@ -31,8 +35,11 @@ def message(*a, force_output=False): caller = getframeinfo(st[0]) slines.append(f"{caller.filename}:{caller.lineno}") cpath = os.path.commonpath(slines) - cs = ";".join([ x[len(cpath)+1:] for x in slines ]) - s = f"[{cpath}|{cs}]\n{s}" + cs = ";".join([x[len(cpath) + 1 :] for x in slines]) + ctx = f"[{cpath}|{cs}]" + if ctx != last_context: + s = f"{ctx}\n{s}" + last_context = ctx if gpt.rank() == 0 or force_output: lines = s.split("\n") if len(lines) > 0: @@ -40,3 +47,27 @@ def message(*a, force_output=False): for line in lines[1:]: print(" :", line) sys.stdout.flush() + + +def backtrace_signal_handler(sig, frame): + + now = datetime.datetime.now() + log_directory = "log/" + now.strftime("%Y-%m-%d") + if not os.path.exists(log_directory): + os.makedirs(log_directory, exist_ok=True) + + log_filename = f"{log_directory}/backtrace.{gpt.rank()}." + now.strftime("%H-%M-%f") + sys.stderr.write(f"Requested GPT backtrace; saved in {log_filename}\n") + sys.stderr.flush() + + fout = open(log_filename, "wt") + + while frame is not None: + caller = getframeinfo(frame) + fout.write(f"{caller.filename}:{caller.lineno}\n") + frame = frame.f_back + + fout.close() + + +signal.signal(signal.SIGUSR2, backtrace_signal_handler)