add termination logging

services/boilerplate/error_logging.py

@@ -0,0 +1,17 @@
+import logging
+import os
+
+
+def write_termination_log(text, log_file="error.log"):
+    """Writes text to termination log.
+
+    Args:
+        text: str
+        log_file: Optional[str]
+    """
+    log_file = os.environ.get("TERMINATION_LOG_FILE", log_file)
+    try:
+        with open(log_file, "a", encoding="utf-8") as handle:
+            handle.write(text)
+    except Exception as e:  # pylint: disable=broad-except
+        logging.warning("Unable to write termination log due to error {}".format(e))

services/finetuning/tsfmfinetuning/.gitignore

@@ -1,6 +1,7 @@
 # THIS FILE IS AUTOMATICALLY GENERATED, YOUR CHANGES WILL BE OVERWRITTEN
 dataframe_checks.py
 dirutil.py
+error_logging.py
 errors.py
 hfutil.py
 inference_payloads.py

services/finetuning/tsfmfinetuning/ftmain.py

@@ -5,6 +5,7 @@
 import yaml
 
 from tsfmfinetuning import TSFM_CONFIG_FILE
+from tsfmfinetuning.error_logging import write_termination_log
 from tsfmfinetuning.finetuning import FinetuningRuntime
 from tsfmfinetuning.ftargs import argparser
 from tsfmfinetuning.ftpayloads import TinyTimeMixerForecastingTuneInput
@@ -38,4 +39,5 @@ def main() -> int:
         exit(main())
     except Exception as e:
         traceback.print_exception(e)
+        write_termination_log(f"Exception when running finetuning {e}")
         exit(1)