diff --git a/.gitignore b/.gitignore index 3c16e466b5..c5bfbd0bc3 100644 --- a/.gitignore +++ b/.gitignore @@ -134,6 +134,7 @@ examples/legacy examples/colocate examples/pset examples/nodeid +examples/client-threaded src/sys/powerpc/atomic-32.s src/sys/powerpc/atomic-64.s diff --git a/examples/Makefile b/examples/Makefile index 07b3d06d18..6ca8603680 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -14,7 +14,7 @@ # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. # Copyright (c) 2013 Mellanox Technologies, Inc. All rights reserved. # Copyright (c) 2016-2020 Intel, Inc. All rights reserved. -# Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. +# Copyright (c) 2021-2024 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -61,7 +61,8 @@ EXAMPLES = \ legacy \ colocate \ pset \ - nodeid + nodeid \ + client-threaded all: $(EXAMPLES) diff --git a/examples/Makefile.include b/examples/Makefile.include index d971d304bc..330c9ecb91 100644 --- a/examples/Makefile.include +++ b/examples/Makefile.include @@ -15,7 +15,7 @@ # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. # Copyright (c) 2013 Mellanox Technologies, Inc. All rights reserved. # Copyright (c) 2016-2020 Intel, Inc. All rights reserved. -# Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. +# Copyright (c) 2021-2024 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -69,4 +69,5 @@ EXTRA_DIST += \ examples/tool.c \ examples/colocate.c \ examples/pset.c \ - examples/nodeid.c + examples/nodeid.c \ + examples/client-threaded.c diff --git a/examples/client-threaded.c b/examples/client-threaded.c new file mode 100644 index 0000000000..3c68316fde --- /dev/null +++ b/examples/client-threaded.c @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006-2013 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. + * Copyright (c) 2021-2024 Nanook Consulting All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +#include + +static pmix_proc_t myproc; + + +void* myThread(void* vargp) +{ + pmix_status_t rc; + pmix_proc_t proc; + pmix_value_t *val; + + PMIX_LOAD_PROCID(&proc, myproc.nspace, 1); + fprintf(stderr, "Fetching remote\n"); + rc = PMIx_Get(&proc, "remote", NULL, 0, &val); + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned: %s(%d)\n", + myproc.nspace, myproc.rank, "foobar", PMIx_Error_string(rc), rc); + return NULL; +} + +int main(int argc, char **argv) +{ + pmix_status_t rc; + pmix_value_t value; + pmix_value_t *val = &value; + char *tmp; + pmix_proc_t proc; + uint32_t nprocs, n, k; + pmix_info_t *info, i2; + bool flag; + pthread_t thread_id[4]; + + /* init us - note that the call to "init" includes the return of + * any job-related info provided by the RM. This includes the + * location of all procs in our job */ + if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %d\n", myproc.nspace, myproc.rank, + rc); + exit(0); + } + fprintf(stderr, "Client ns %s rank %d Running\n", myproc.nspace, myproc.rank); + + value.type = PMIX_UINT64; + value.data.uint64 = 1234; + if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_LOCAL, "local", &value))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Put internal failed: %d\n", myproc.nspace, + myproc.rank, rc); + goto done; + } + + value.type = PMIX_UINT64; + value.data.uint64 = 5678; + if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_REMOTE, "remote", &value))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Put internal failed: %d\n", myproc.nspace, + myproc.rank, rc); + goto done; + } + + /* push the data to our PMIx server */ + if (PMIX_SUCCESS != (rc = PMIx_Commit())) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Commit failed: %d\n", myproc.nspace, + myproc.rank, rc); + goto done; + } + + /* call fence to synchronize with our peers - instruct + * the fence operation to collect and return all "put" + * data from our peers */ + PMIX_INFO_CREATE(info, 1); + flag = true; + PMIX_INFO_LOAD(info, PMIX_COLLECT_DATA, &flag, PMIX_BOOL); + PMIX_LOAD_PROCID(&proc, myproc.nspace, PMIX_RANK_WILDCARD); + if (PMIX_SUCCESS != (rc = PMIx_Fence(&proc, 1, info, 1))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Fence failed: %d\n", myproc.nspace, myproc.rank, + rc); + goto done; + } + PMIX_INFO_FREE(info, 1); + + // spawn a few threads - each thread attempts to get data + // published by a peer + if (0 == myproc.rank) { + fprintf(stderr, "%u: SPAWNING THREADS\n", myproc.rank); + for (n=0; n < 4; n++) { + pthread_create(&thread_id[n], NULL, myThread, NULL); + } + // collect them all + for (n=0; n < 4; n++) { + pthread_join(thread_id[n], NULL); + } + } + +done: + /* finalize us */ + rc = PMIx_Finalize(NULL, 0); + if (PMIX_SUCCESS != rc) { + fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, + myproc.rank, rc); + } + fflush(stderr); + return (0); +} diff --git a/src/mca/schizo/ompi/schizo_ompi.c b/src/mca/schizo/ompi/schizo_ompi.c index ac926f3ea1..3fb46e7ca6 100644 --- a/src/mca/schizo/ompi/schizo_ompi.c +++ b/src/mca/schizo/ompi/schizo_ompi.c @@ -1847,6 +1847,242 @@ static int parse_env(char **srcenv, char ***dstenv, return PRTE_SUCCESS; } +// NOTE: This code is fundamentally the same (module PMIX <-> OPAL) +// as the translate_params() routine in the OMPI repo's +// opal/mca/pmix/base/pmix_base_fns.c file. If there are +// changes here, there are likely to be changes there. + +static bool check_prte_overlap(char *var, char *value) +{ + char *tmp; + + if (0 == strncmp(var, "dl_", 3)) { + pmix_asprintf(&tmp, "PRTE_MCA_prtedl_%s", &var[3]); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, value, false); + free(tmp); + return true; + } else if (0 == strncmp(var, "oob_", 4)) { + pmix_asprintf(&tmp, "PRTE_MCA_%s", var); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, value, false); + free(tmp); + return true; + } else if (0 == strncmp(var, "hwloc_", 6)) { + pmix_asprintf(&tmp, "PRTE_MCA_%s", var); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, value, false); + free(tmp); + return true; + } else if (0 == strncmp(var, "if_", 3)) { + // need to convert if to prteif + pmix_asprintf(&tmp, "PRTE_MCA_prteif_%s", &var[3]); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, value, false); + free(tmp); + return true; + } else if (0 == strncmp(var, "reachable_", strlen("reachable_"))) { + // need to convert reachable to prtereachable + pmix_asprintf(&tmp, "PRTE_MCA_prtereachable_%s", &var[strlen("reachable_")]); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, value, false); + free(tmp); + return true; + } else if (0 == strncmp(var, "plm_rsh_", strlen("plm_rsh_"))) { + // need to convert rsh to ssh + pmix_asprintf(&tmp, "PRTE_MCA_plm_ssh_%s", &var[strlen("plm_rsh_")]); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, value, false); + free(tmp); + return true; + } else if (0 == strncmp(var, "orte_", strlen("orte_"))) { + // need to convert "orte" to "prte" + pmix_asprintf(&tmp, "PRTE_MCA_prte_%s", &var[strlen("orte_")]); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, value, false); + free(tmp); + return true; + } + return false; +} + +static bool check_pmix_overlap(char *var, char *value) +{ + char *tmp; + + if (0 == strncmp(var, "dl_", 3)) { + pmix_asprintf(&tmp, "PMIX_MCA_pdl_%s", &var[3]); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, value, false); + free(tmp); + return true; + } else if (0 == strncmp(var, "oob_", 4)) { + pmix_asprintf(&tmp, "PMIX_MCA_ptl_%s", &var[4]); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, value, false); + free(tmp); + return true; + } else if (0 == strncmp(var, "hwloc_", 6)) { + pmix_asprintf(&tmp, "PMIX_MCA_%s", var); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, value, false); + free(tmp); + return true; + } else if (0 == strncmp(var, "if_", 3)) { + // need to convert if to pif + pmix_asprintf(&tmp, "PMIX_MCA_pif_%s", &var[3]); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, value, false); + free(tmp); + return true; + } + return false; +} + +static int translate_params(void) +{ + char *evar, *tmp, *e2; + char *file; + const char *home; + pmix_list_t params; + pmix_mca_base_var_file_value_t *fv; + uid_t uid; + int n, len; + + /* since we are the proxy, we need to check the OMPI default + * MCA params to see if there is something relating to PRRTE + * in them - this would be "old" references to things from + * ORTE, as well as a few OPAL references that also impact us + * + * NOTE: we do this in the following precedence order. Note + * that we do not overwrite at any step - this is so that we + * don't overwrite something previously set by the user. So + * the order to execution is the opposite of the intended + * precedence order. + * + * 1. check the environmental paramaters for OMPI_MCA values + * that need to be translated + * + * 2. the user's home directory file as it should + * overwrite the system default file, but not the + * envars + * + * 3. the system default parameter file + */ + len = strlen("OMPI_MCA_"); + for (n=0; NULL != environ[n]; n++) { + if (0 == strncmp(environ[n], "OMPI_MCA_", len)) { + e2 = strdup(environ[n]); + evar = strrchr(e2, '='); + *evar = '\0'; + ++evar; + if (check_prte_overlap(&e2[len], evar)) { + // check for pmix overlap + check_pmix_overlap(&e2[len], evar); + } else if (pmix_pmdl_base_check_prte_param(&e2[len])) { + pmix_asprintf(&tmp, "PRTE_MCA_%s", &e2[len]); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, evar, false); + free(tmp); + // check for pmix overlap + check_pmix_overlap(&e2[len], evar); + } else if (pmix_pmdl_base_check_pmix_param(&e2[len])) { + pmix_asprintf(&tmp, "PMIX_MCA_%s", &e2[len]); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, evar, false); + free(tmp); + } + free(e2); + } + } + + /* see if the user has a default MCA param file */ + uid = geteuid(); + + /* try to get their home directory */ + home = pmix_home_directory(uid); + if (NULL != home) { + file = pmix_os_path(false, home, ".openmpi", "mca-params.conf", NULL); + PMIX_CONSTRUCT(¶ms, pmix_list_t); + pmix_mca_base_parse_paramfile(file, ¶ms); + free(file); + PMIX_LIST_FOREACH (fv, ¶ms, pmix_mca_base_var_file_value_t) { + // see if this param relates to PRRTE + if (check_prte_overlap(fv->mbvfv_var, fv->mbvfv_value)) { + check_pmix_overlap(fv->mbvfv_var, fv->mbvfv_value); + } else if (pmix_pmdl_base_check_prte_param(fv->mbvfv_var)) { + pmix_asprintf(&tmp, "PRTE_MCA_%s", fv->mbvfv_var); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, fv->mbvfv_value, false); + free(tmp); + // if this relates to the DL, OOB, HWLOC, IF, or + // REACHABLE frameworks, then we also need to set + // the equivalent PMIx value + check_pmix_overlap(fv->mbvfv_var, fv->mbvfv_value); + } else if (pmix_pmdl_base_check_pmix_param(fv->mbvfv_var)) { + pmix_asprintf(&tmp, "PMIX_MCA_%s", fv->mbvfv_var); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, fv->mbvfv_value, false); + free(tmp); + } + } + PMIX_LIST_DESTRUCT(¶ms); + } + + /* check if the user has set OMPIHOME in their environment */ + if (NULL != (evar = getenv("OMPIHOME"))) { + /* look for the default MCA param file */ + file = pmix_os_path(false, evar, "etc", "openmpi-mca-params.conf", NULL); + PMIX_CONSTRUCT(¶ms, pmix_list_t); + pmix_mca_base_parse_paramfile(file, ¶ms); + free(file); + PMIX_LIST_FOREACH (fv, ¶ms, pmix_mca_base_var_file_value_t) { + // see if this param overlaps with PRRTE + check_prte_overlap(fv->mbvfv_var, fv->mbvfv_value); + // see if it overlaps with PMIx + check_pmix_overlap(fv->mbvfv_var, fv->mbvfv_value); + // see if it relates to PRRTE + if (pmix_pmdl_base_check_prte_param(fv->mbvfv_var)) { + pmix_asprintf(&tmp, "PRTE_MCA_%s", fv->mbvfv_var); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, fv->mbvfv_value, false); + free(tmp); + // if this relates to the DL, OOB, HWLOC, IF, or + // REACHABLE frameworks, then we also need to set + // the equivalent PMIx value + check_pmix_overlap(fv->mbvfv_var, fv->mbvfv_value); + } + // see if it relates to PMIx + if (pmix_pmdl_base_check_pmix_param(fv->mbvfv_var)) { + pmix_asprintf(&tmp, "PMIX_MCA_%s", fv->mbvfv_var); + // set it, but don't overwrite if they already + // have a value in our environment + setenv(tmp, fv->mbvfv_value, false); + free(tmp); + } + } + PMIX_LIST_DESTRUCT(¶ms); + } + + return 100; +} + static int detect_proxy(char *personalities) { char *evar; @@ -1862,7 +2098,7 @@ static int detect_proxy(char *personalities) /* this is a list of personalities we need to check - * if it contains "ompi", then we are available */ if (NULL != strstr(personalities, "ompi")) { - return 100; + return translate_params(); } return 0; }