Implement the PMIX_JOB_CHILD_SEP support

Provide an optional way to determine the fate of
child jobs if/when the parent job terminates. In
the current implementation, we do not support
continuation after parent job abnormally
terminates - has to be a normal termination.

Users can toggle the behavior by providing
the PMIX_JOB_CHILD_SEP attribute in their
job info passed to PMIx_Spawn. In the absence
of that attribute, we default to allowing the
child to continue executing.

Provide an output when child jobs are terminated
warning the user that this has happened, and why.

Signed-off-by: Ralph Castain <[email protected]>

src/docs/show-help-files/help-state-base.txt

@@ -40,3 +40,20 @@ behavior:
 
 You must specify one of the above in combination with NOTIFYERRORS in order
 to receive notifications of errors. Please correct the situation and try again.
+#
+[child-term]
+At least one child job is being terminated due to termination of
+its parent:
+
+  Parent: %s
+  Child:  %s
+
+This behavior is controlled by setting the PMIX_JOB_CHILD_SEP attribute
+in the job info provided at time of spawn for the child job. When set to
+"true", the runtime will "separate" the child from its parent and allow
+it to continue execution after parent termination. Note that this is only
+true for parents that normally terminate - abnormal termination will always
+result in a complete teardown of all child jobs.
+
+In the absence of the attribute, the runtime will default to the "true"
+behavior.

src/mca/state/base/help-state-base.txt

@@ -3,7 +3,7 @@
 # Copyright (c) 2018      Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
 # Copyright (c) 2019      Intel, Inc.  All rights reserved.
-# Copyright (c) 2022      Nanook Consulting.  All rights reserved.
+# Copyright (c) 2022-2024 Nanook Consulting  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -40,3 +40,20 @@ behavior:
 
 You must specify one of the above in combination with NOTIFYERRORS in order
 to receive notifications of errors. Please correct the situation and try again.
+#
+[child-term]
+At least one child job is being terminated due to termination of
+its parent:
+
+  Parent: %s
+  Child:  %s
+
+This behavior is controlled by setting the PMIX_JOB_CHILD_SEP attribute
+in the job info provided at time of spawn for the child job. When set to
+"true", the runtime will "separate" the child from its parent and allow
+it to continue execution after parent termination. Note that this is only
+true for parents that normally terminate - abnormal termination will always
+result in a complete teardown of all child jobs.
+
+In the absence of the attribute, the runtime will default to the "true"
+behavior.

src/mca/state/dvm/state_dvm.c

@@ -29,6 +29,7 @@
 #include "src/util/pmix_output.h"
 #include "src/util/proc_info.h"
 #include "src/util/session_dir.h"
+#include "src/util/pmix_show_help.h"
 
 #include "src/mca/errmgr/errmgr.h"
 #include "src/mca/filem/filem.h"
@@ -507,7 +508,7 @@ static void check_complete(int fd, short args, void *cbdata)
     prte_session_t *session;
     prte_job_t *jdata, *jptr;
     prte_proc_t *proc;
-    int i, rc;
+    int i, rc, nprocs;
     prte_node_t *node;
     prte_job_map_t *map;
     int32_t index;
@@ -521,7 +522,7 @@ static void check_complete(int fd, short args, void *cbdata)
     hwloc_obj_t obj;
     hwloc_obj_type_t type;
     hwloc_cpuset_t boundcpus, tgt;
-    bool takeall;
+    bool takeall, sep, *sepptr = &sep;
     PRTE_HIDE_UNUSED_PARAMS(fd, args);
 
     PMIX_ACQUIRE_OBJECT(caddy);
@@ -824,21 +825,33 @@ static void check_complete(int fd, short args, void *cbdata)
         prte_state_base_check_fds(jdata);
     }
 
-    /* if this job was a launcher, then we need to abort all of its
-     * child jobs that might still be running */
+    /* if this job started child jobs, then we need to abort all of its
+     * child jobs that might still be running unless designated to
+     * run independently of their parent */
     if (0 < pmix_list_get_size(&jdata->children)) {
         PMIX_CONSTRUCT(&procs, pmix_pointer_array_t);
         pmix_pointer_array_init(&procs, 1, INT_MAX, 1);
+        nprocs = 0;
         PMIX_LIST_FOREACH(jptr, &jdata->children, prte_job_t)
         {
-            proc = PMIX_NEW(prte_proc_t);
-            PMIX_LOAD_PROCID(&proc->name, jptr->nspace, PMIX_RANK_WILDCARD);
-            pmix_pointer_array_add(&procs, proc);
+            if (prte_get_attribute(&jptr->attributes, PRTE_JOB_CHILD_SEP, (void**)&sepptr, PMIX_BOOL) && !sep) {
+                proc = PMIX_NEW(prte_proc_t);
+                PMIX_LOAD_PROCID(&proc->name, jptr->nspace, PMIX_RANK_WILDCARD);
+                pmix_pointer_array_add(&procs, proc);
+                ++nprocs;
+                if (1 == nprocs) {
+                    // output a warning message that at least one child is being terminated
+                    pmix_show_help("help-state-base.txt", "child-term", true,
+                                   jdata->nspace, jptr->nspace);
+                }
+            }
         }
-        prte_plm.terminate_procs(&procs);
-        for (i = 0; i < procs.size; i++) {
-            if (NULL != (proc = (prte_proc_t *) pmix_pointer_array_get_item(&procs, i))) {
-                PMIX_RELEASE(proc);
+        if (0 < nprocs) {
+            prte_plm.terminate_procs(&procs);
+            for (i = 0; i < procs.size; i++) {
+                if (NULL != (proc = (prte_proc_t *) pmix_pointer_array_get_item(&procs, i))) {
+                    PMIX_RELEASE(proc);
+                }
             }
         }
         PMIX_DESTRUCT(&procs);

src/prted/pmix/pmix_server_dyn.c

@@ -368,6 +368,12 @@ int prte_pmix_xfer_job_info(prte_job_t *jdata,
             prte_set_attribute(&jdata->attributes, PRTE_JOB_CONTINUOUS, PRTE_ATTR_GLOBAL,
                                &flag, PMIX_BOOL);
 
+            /*** CHILD INDEPENDENCE  ***/
+        } else if (PMIX_CHECK_KEY(info, PMIX_SPAWN_CHILD_SEP)) {
+            flag = PMIX_INFO_TRUE(info);
+            prte_set_attribute(&jdata->attributes, PRTE_JOB_CHILD_SEP, PRTE_ATTR_GLOBAL,
+                               &flag, PMIX_BOOL);
+
             /***   MAX RESTARTS  ***/
         } else if (PMIX_CHECK_KEY(info, PMIX_MAX_RESTARTS)) {
             for (i = 0; i < jdata->apps->size; i++) {

src/util/attr.c

@@ -507,6 +507,8 @@ const char *prte_attr_key_to_str(prte_attribute_key_t key)
             return "ALLOC REF ID";
         case PRTE_JOB_BINDING_LIMIT:
             return "JOB BINDING LIMIT";
+        case PRTE_JOB_CHILD_SEP:
+            return "CHILD SEP";
 
         case PRTE_PROC_NOBARRIER:
             return "PROC-NOBARRIER";

src/util/attr.h

@@ -228,6 +228,9 @@ typedef uint16_t prte_job_flags_t;
                                                                        //         from the request
 #define PRTE_JOB_BINDING_LIMIT              (PRTE_JOB_START_KEY + 115) // (uint16_t) - Max number of procs to bind to specified
                                                                        //          target type before moving to next target
+#define PRTE_JOB_CHILD_SEP                  (PRTE_JOB_START_KEY + 116) // bool - child job is to be considered independent
+                                                                       //        from its parent, do not terminate if
+                                                                       //        parent dies first
 
 #define PRTE_JOB_MAX_KEY (PRTE_JOB_START_KEY + 200)