From bdd142ae243bf076c94a9a9ed5024553da756ddf Mon Sep 17 00:00:00 2001
From: Stuart Siegel <12914116+ssiegel95@users.noreply.github.com>
Date: Tue, 10 Dec 2024 21:50:39 -0500
Subject: [PATCH] add termination logging

---
 services/boilerplate/error_logging.py         | 17 +++++++++++++++++
 services/finetuning/tsfmfinetuning/.gitignore |  1 +
 services/finetuning/tsfmfinetuning/ftmain.py  |  2 ++
 3 files changed, 20 insertions(+)
 create mode 100644 services/boilerplate/error_logging.py

diff --git a/services/boilerplate/error_logging.py b/services/boilerplate/error_logging.py
new file mode 100644
index 00000000..abdcc20d
--- /dev/null
+++ b/services/boilerplate/error_logging.py
@@ -0,0 +1,17 @@
+import logging
+import os
+
+
+def write_termination_log(text, log_file="error.log"):
+    """Writes text to termination log.
+
+    Args:
+        text: str
+        log_file: Optional[str]
+    """
+    log_file = os.environ.get("TERMINATION_LOG_FILE", log_file)
+    try:
+        with open(log_file, "a", encoding="utf-8") as handle:
+            handle.write(text)
+    except Exception as e:  # pylint: disable=broad-except
+        logging.warning("Unable to write termination log due to error {}".format(e))
diff --git a/services/finetuning/tsfmfinetuning/.gitignore b/services/finetuning/tsfmfinetuning/.gitignore
index fc766639..5dda1868 100644
--- a/services/finetuning/tsfmfinetuning/.gitignore
+++ b/services/finetuning/tsfmfinetuning/.gitignore
@@ -1,6 +1,7 @@
 # THIS FILE IS AUTOMATICALLY GENERATED, YOUR CHANGES WILL BE OVERWRITTEN
 dataframe_checks.py
 dirutil.py
+error_logging.py
 errors.py
 hfutil.py
 inference_payloads.py
diff --git a/services/finetuning/tsfmfinetuning/ftmain.py b/services/finetuning/tsfmfinetuning/ftmain.py
index c0d57aac..d90e64d5 100644
--- a/services/finetuning/tsfmfinetuning/ftmain.py
+++ b/services/finetuning/tsfmfinetuning/ftmain.py
@@ -5,6 +5,7 @@
 import yaml
 
 from tsfmfinetuning import TSFM_CONFIG_FILE
+from tsfmfinetuning.error_logging import write_termination_log
 from tsfmfinetuning.finetuning import FinetuningRuntime
 from tsfmfinetuning.ftargs import argparser
 from tsfmfinetuning.ftpayloads import TinyTimeMixerForecastingTuneInput
@@ -38,4 +39,5 @@ def main() -> int:
         exit(main())
     except Exception as e:
         traceback.print_exception(e)
+        write_termination_log(f"Exception when running finetuning {e}")
         exit(1)