From 8795110dc4ff34e866b0c8c579855e6d98c75986 Mon Sep 17 00:00:00 2001
From: Ralph Castain <rhc@pmix.org>
Date: Fri, 18 Oct 2024 07:43:23 -0600
Subject: [PATCH] Implement the PMIX_JOB_CHILD_SEP support

Provide an optional way to determine the fate of
child jobs if/when the parent job terminates. In
the current implementation, we do not support
continuation after parent job abnormally
terminates - has to be a normal termination.

Users can toggle the behavior by providing
the PMIX_JOB_CHILD_SEP attribute in their
job info passed to PMIx_Spawn. In the absence
of that attribute, we default to allowing the
child to continue executing.

Signed-off-by: Ralph Castain <rhc@pmix.org>
---
 src/mca/state/dvm/state_dvm.c    | 29 ++++++++++++++++++-----------
 src/prted/pmix/pmix_server_dyn.c |  6 ++++++
 src/util/attr.c                  |  2 ++
 src/util/attr.h                  |  3 +++
 4 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/src/mca/state/dvm/state_dvm.c b/src/mca/state/dvm/state_dvm.c
index cd4d0f8811..b0e9b89c53 100644
--- a/src/mca/state/dvm/state_dvm.c
+++ b/src/mca/state/dvm/state_dvm.c
@@ -507,7 +507,7 @@ static void check_complete(int fd, short args, void *cbdata)
     prte_session_t *session;
     prte_job_t *jdata, *jptr;
     prte_proc_t *proc;
-    int i, rc;
+    int i, rc, nprocs;
     prte_node_t *node;
     prte_job_map_t *map;
     int32_t index;
@@ -521,7 +521,7 @@ static void check_complete(int fd, short args, void *cbdata)
     hwloc_obj_t obj;
     hwloc_obj_type_t type;
     hwloc_cpuset_t boundcpus, tgt;
-    bool takeall;
+    bool takeall, sep, *sepptr = &sep;
     PRTE_HIDE_UNUSED_PARAMS(fd, args);
 
     PMIX_ACQUIRE_OBJECT(caddy);
@@ -824,21 +824,28 @@ static void check_complete(int fd, short args, void *cbdata)
         prte_state_base_check_fds(jdata);
     }
 
-    /* if this job was a launcher, then we need to abort all of its
-     * child jobs that might still be running */
+    /* if this job started child jobs, then we need to abort all of its
+     * child jobs that might still be running unless designated to
+     * run independently of their parent */
     if (0 < pmix_list_get_size(&jdata->children)) {
         PMIX_CONSTRUCT(&procs, pmix_pointer_array_t);
         pmix_pointer_array_init(&procs, 1, INT_MAX, 1);
+        nprocs = 0;
         PMIX_LIST_FOREACH(jptr, &jdata->children, prte_job_t)
         {
-            proc = PMIX_NEW(prte_proc_t);
-            PMIX_LOAD_PROCID(&proc->name, jptr->nspace, PMIX_RANK_WILDCARD);
-            pmix_pointer_array_add(&procs, proc);
+            if (prte_get_attribute(&jptr->attributes, PRTE_JOB_CHILD_SEP, (void**)&sepptr, PMIX_BOOL) && !sep) {
+                proc = PMIX_NEW(prte_proc_t);
+                PMIX_LOAD_PROCID(&proc->name, jptr->nspace, PMIX_RANK_WILDCARD);
+                pmix_pointer_array_add(&procs, proc);
+                ++nprocs;
+            }
         }
-        prte_plm.terminate_procs(&procs);
-        for (i = 0; i < procs.size; i++) {
-            if (NULL != (proc = (prte_proc_t *) pmix_pointer_array_get_item(&procs, i))) {
-                PMIX_RELEASE(proc);
+        if (0 < nprocs) {
+            prte_plm.terminate_procs(&procs);
+            for (i = 0; i < procs.size; i++) {
+                if (NULL != (proc = (prte_proc_t *) pmix_pointer_array_get_item(&procs, i))) {
+                    PMIX_RELEASE(proc);
+                }
             }
         }
         PMIX_DESTRUCT(&procs);
diff --git a/src/prted/pmix/pmix_server_dyn.c b/src/prted/pmix/pmix_server_dyn.c
index 6865126c56..ee3b232ebe 100644
--- a/src/prted/pmix/pmix_server_dyn.c
+++ b/src/prted/pmix/pmix_server_dyn.c
@@ -368,6 +368,12 @@ int prte_pmix_xfer_job_info(prte_job_t *jdata,
             prte_set_attribute(&jdata->attributes, PRTE_JOB_CONTINUOUS, PRTE_ATTR_GLOBAL,
                                &flag, PMIX_BOOL);
 
+            /*** CHILD INDEPENDENCE  ***/
+        } else if (PMIX_CHECK_KEY(info, PMIX_SPAWN_CHILD_SEP)) {
+            flag = PMIX_INFO_TRUE(info);
+            prte_set_attribute(&jdata->attributes, PRTE_JOB_CHILD_SEP, PRTE_ATTR_GLOBAL,
+                               &flag, PMIX_BOOL);
+
             /***   MAX RESTARTS  ***/
         } else if (PMIX_CHECK_KEY(info, PMIX_MAX_RESTARTS)) {
             for (i = 0; i < jdata->apps->size; i++) {
diff --git a/src/util/attr.c b/src/util/attr.c
index 2e929f3d08..7f1d3ec5dd 100644
--- a/src/util/attr.c
+++ b/src/util/attr.c
@@ -507,6 +507,8 @@ const char *prte_attr_key_to_str(prte_attribute_key_t key)
             return "ALLOC REF ID";
         case PRTE_JOB_BINDING_LIMIT:
             return "JOB BINDING LIMIT";
+        case PRTE_JOB_CHILD_SEP:
+            return "CHILD SEP";
 
         case PRTE_PROC_NOBARRIER:
             return "PROC-NOBARRIER";
diff --git a/src/util/attr.h b/src/util/attr.h
index d583f5190d..b7eb2d227d 100644
--- a/src/util/attr.h
+++ b/src/util/attr.h
@@ -228,6 +228,9 @@ typedef uint16_t prte_job_flags_t;
                                                                        //         from the request
 #define PRTE_JOB_BINDING_LIMIT              (PRTE_JOB_START_KEY + 115) // (uint16_t) - Max number of procs to bind to specified
                                                                        //          target type before moving to next target
+#define PRTE_JOB_CHILD_SEP                  (PRTE_JOB_START_KEY + 116) // bool - child job is to be considered independent
+                                                                       //        from its parent, do not terminate if
+                                                                       //        parent dies first
 
 #define PRTE_JOB_MAX_KEY (PRTE_JOB_START_KEY + 200)