Skip to content

Commit

Permalink
Implement the PMIX_JOB_CHILD_SEP support
Browse files Browse the repository at this point in the history
Provide an optional way to determine the fate of
child jobs if/when the parent job terminates. In
the current implementation, we do not support
continuation after parent job abnormally
terminates - has to be a normal termination.

Users can toggle the behavior by providing
the PMIX_JOB_CHILD_SEP attribute in their
job info passed to PMIx_Spawn. In the absence
of that attribute, we default to allowing the
child to continue executing.

Signed-off-by: Ralph Castain <[email protected]>
  • Loading branch information
rhc54 committed Oct 18, 2024
1 parent fcbefcc commit 8795110
Show file tree
Hide file tree
Showing 4 changed files with 29 additions and 11 deletions.
29 changes: 18 additions & 11 deletions src/mca/state/dvm/state_dvm.c
Original file line number Diff line number Diff line change
Expand Up @@ -507,7 +507,7 @@ static void check_complete(int fd, short args, void *cbdata)
prte_session_t *session;
prte_job_t *jdata, *jptr;
prte_proc_t *proc;
int i, rc;
int i, rc, nprocs;
prte_node_t *node;
prte_job_map_t *map;
int32_t index;
Expand All @@ -521,7 +521,7 @@ static void check_complete(int fd, short args, void *cbdata)
hwloc_obj_t obj;
hwloc_obj_type_t type;
hwloc_cpuset_t boundcpus, tgt;
bool takeall;
bool takeall, sep, *sepptr = &sep;
PRTE_HIDE_UNUSED_PARAMS(fd, args);

PMIX_ACQUIRE_OBJECT(caddy);
Expand Down Expand Up @@ -824,21 +824,28 @@ static void check_complete(int fd, short args, void *cbdata)
prte_state_base_check_fds(jdata);
}

/* if this job was a launcher, then we need to abort all of its
* child jobs that might still be running */
/* if this job started child jobs, then we need to abort all of its
* child jobs that might still be running unless designated to
* run independently of their parent */
if (0 < pmix_list_get_size(&jdata->children)) {
PMIX_CONSTRUCT(&procs, pmix_pointer_array_t);
pmix_pointer_array_init(&procs, 1, INT_MAX, 1);
nprocs = 0;
PMIX_LIST_FOREACH(jptr, &jdata->children, prte_job_t)
{
proc = PMIX_NEW(prte_proc_t);
PMIX_LOAD_PROCID(&proc->name, jptr->nspace, PMIX_RANK_WILDCARD);
pmix_pointer_array_add(&procs, proc);
if (prte_get_attribute(&jptr->attributes, PRTE_JOB_CHILD_SEP, (void**)&sepptr, PMIX_BOOL) && !sep) {
proc = PMIX_NEW(prte_proc_t);
PMIX_LOAD_PROCID(&proc->name, jptr->nspace, PMIX_RANK_WILDCARD);
pmix_pointer_array_add(&procs, proc);
++nprocs;
}
}
prte_plm.terminate_procs(&procs);
for (i = 0; i < procs.size; i++) {
if (NULL != (proc = (prte_proc_t *) pmix_pointer_array_get_item(&procs, i))) {
PMIX_RELEASE(proc);
if (0 < nprocs) {
prte_plm.terminate_procs(&procs);
for (i = 0; i < procs.size; i++) {
if (NULL != (proc = (prte_proc_t *) pmix_pointer_array_get_item(&procs, i))) {
PMIX_RELEASE(proc);
}
}
}
PMIX_DESTRUCT(&procs);
Expand Down
6 changes: 6 additions & 0 deletions src/prted/pmix/pmix_server_dyn.c
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,12 @@ int prte_pmix_xfer_job_info(prte_job_t *jdata,
prte_set_attribute(&jdata->attributes, PRTE_JOB_CONTINUOUS, PRTE_ATTR_GLOBAL,
&flag, PMIX_BOOL);

/*** CHILD INDEPENDENCE ***/
} else if (PMIX_CHECK_KEY(info, PMIX_SPAWN_CHILD_SEP)) {
flag = PMIX_INFO_TRUE(info);
prte_set_attribute(&jdata->attributes, PRTE_JOB_CHILD_SEP, PRTE_ATTR_GLOBAL,
&flag, PMIX_BOOL);

/*** MAX RESTARTS ***/
} else if (PMIX_CHECK_KEY(info, PMIX_MAX_RESTARTS)) {
for (i = 0; i < jdata->apps->size; i++) {
Expand Down
2 changes: 2 additions & 0 deletions src/util/attr.c
Original file line number Diff line number Diff line change
Expand Up @@ -507,6 +507,8 @@ const char *prte_attr_key_to_str(prte_attribute_key_t key)
return "ALLOC REF ID";
case PRTE_JOB_BINDING_LIMIT:
return "JOB BINDING LIMIT";
case PRTE_JOB_CHILD_SEP:
return "CHILD SEP";

case PRTE_PROC_NOBARRIER:
return "PROC-NOBARRIER";
Expand Down
3 changes: 3 additions & 0 deletions src/util/attr.h
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,9 @@ typedef uint16_t prte_job_flags_t;
// from the request
#define PRTE_JOB_BINDING_LIMIT (PRTE_JOB_START_KEY + 115) // (uint16_t) - Max number of procs to bind to specified
// target type before moving to next target
#define PRTE_JOB_CHILD_SEP (PRTE_JOB_START_KEY + 116) // bool - child job is to be considered independent
// from its parent, do not terminate if
// parent dies first

#define PRTE_JOB_MAX_KEY (PRTE_JOB_START_KEY + 200)

Expand Down

0 comments on commit 8795110

Please sign in to comment.