From c47b429ec114539dfddf0bced5ef665525024b08 Mon Sep 17 00:00:00 2001 From: "Brian C. Van Essen" Date: Tue, 27 Aug 2024 11:15:30 -0700 Subject: [PATCH] If a python virtual enviornment was defined and used during the build time, the Lua module file will now activate it when loaded. Removed the TCL module file since it wasn't being used by systems. Added a prompt name to the python venv. Fixed an empty variable field in the Lassen gitlab code that deleted other variables. --- .gitlab/build-and-test-lassen.yml | 1 - .gitlab/setup_env.sh | 2 +- CMakeLists.txt | 33 ++++----- cmake/configure_files/lbann_module.lua.in | 10 ++- cmake/configure_files/lbann_module.tcl.in | 90 ----------------------- 5 files changed, 22 insertions(+), 114 deletions(-) delete mode 100644 cmake/configure_files/lbann_module.tcl.in diff --git a/.gitlab/build-and-test-lassen.yml b/.gitlab/build-and-test-lassen.yml index 9a5602db3e..d76c0af97e 100644 --- a/.gitlab/build-and-test-lassen.yml +++ b/.gitlab/build-and-test-lassen.yml @@ -52,5 +52,4 @@ clang-16-0-6-gcc-11-2-1-cuda-12-2-2-distconv-lassen: .build-and-test-on-lassen: stage: build tags: [lassen, batch] - variables: extends: .build-and-test diff --git a/.gitlab/setup_env.sh b/.gitlab/setup_env.sh index f5d7d661cd..f3cde94b44 100644 --- a/.gitlab/setup_env.sh +++ b/.gitlab/setup_env.sh @@ -186,7 +186,7 @@ echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" export LBANN_PYTHON_VENV_DIR="${INSTALL_EXTERNALS_ROOT}/${SYSTEM_INSTALL_PREFIX_EXTERNALS}/venv" if [[ ! -e "${LBANN_PYTHON_VENV_DIR}/pyvenv.cfg" ]]; then - CMD="python3 -m venv ${LBANN_PYTHON_VENV_DIR}" + CMD="python3 -m venv --prompt lbann-venv ${LBANN_PYTHON_VENV_DIR}" echo "${CMD}" ${CMD} fi diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c880d11f7..3030cde812 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1207,31 +1207,26 @@ if (DEFINED ENV{LBANN_PYTHON_VENV_DIR}) else () set(LBANN_PYTHON_VENV_DIR "" CACHE STRING "") endif() -message("BVE I have found a variable >${LBANN_PYTHON_VENV_DIR}<") configure_file( "${CMAKE_SOURCE_DIR}/cmake/configure_files/lbann_module.lua.in" "${CMAKE_BINARY_DIR}/lbann_module.lua.install" @ONLY) -configure_file( - "${CMAKE_SOURCE_DIR}/cmake/configure_files/lbann_module.tcl.in" - "${CMAKE_BINARY_DIR}/lbann_module.tcl.install") install(FILES "${CMAKE_BINARY_DIR}/lbann_module.lua.install" RENAME "${LBANN_MODULEFILE_NAME}" DESTINATION "${CMAKE_INSTALL_SYSCONFDIR}/modulefiles/lbann") -install(FILES "${CMAKE_BINARY_DIR}/lbann_module.tcl.install" - RENAME "${LBANN_VERSION}" - DESTINATION "${CMAKE_INSTALL_SYSCONFDIR}/modulefiles/lbann") -message("BVE ready to install with VENV variable >${LBANN_PYTHON_VENV_DIR}<") -message("BVE ready to install with VENV variable >$ENV{LBANN_PYTHON_VENV_DIR}<") -install(CODE [===[ - message("\n********************************************************************************") - message("LBANN is installed into ${CMAKE_INSTALL_PREFIX} - to load the LBANN module:") - message(" module use ${CMAKE_INSTALL_PREFIX}/etc/modulefiles") - message(" module load lbann") - message(" source ${LBANN_PYTHON_VENV_DIR}/bin/activate") - message(" source $ENV{LBANN_PYTHON_VENV_DIR}/bin/activate") - message(" source \${LBANN_PYTHON_VENV_DIR}/bin/activate") - message("********************************************************************************") -]===]) + +if (DEFINED ENV{LBANN_PYTHON_VENV_DIR}) +set(_PYTHON_VENV_INSTALL_MSG +"\nA Python virtual environment was used to build and install LBANN. + Loading the module will activate the environment in: ${LBANN_PYTHON_VENV_DIR}") +endif () + +install(CODE " + message(\"\n********************************************************************************\") + message(\"LBANN is installed into ${CMAKE_INSTALL_PREFIX} - to load the LBANN module:\") + message(\" module use ${CMAKE_INSTALL_PREFIX}/etc/modulefiles\") + message(\" module load lbann ${_PYTHON_VENV_INSTALL_MSG}\") + message(\"********************************************************************************\") +") diff --git a/cmake/configure_files/lbann_module.lua.in b/cmake/configure_files/lbann_module.lua.in index a1c7744f5f..c9162535aa 100644 --- a/cmake/configure_files/lbann_module.lua.in +++ b/cmake/configure_files/lbann_module.lua.in @@ -90,9 +90,13 @@ prepend_path("LD_LIBRARY_PATH","@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@") prepend_path("PYTHONPATH","@PYTHON_INSTALL_PREFIX@/@CMAKE_INSTALL_PYTHONDIR@") prepend_path("LBANN_PYTHON_SITE_PACKAGES","@PYTHON_INSTALL_PREFIX@/@CMAKE_INSTALL_PYTHONDIR@") -pushenv("LBANN_PYTHON_VENV_DIR","@LBANN_PYTHON_VENV_DIR@") -pushenv("LBANN_PYTHON_VENV_SITE_PACKAGES","@LBANN_PYTHON_VENV_DIR@/@CMAKE_INSTALL_PYTHONDIR@") -prepend_path("PYTHONPATH","@LBANN_PYTHON_VENV_DIR@/@CMAKE_INSTALL_PYTHONDIR@") +if ("@LBANN_PYTHON_VENV_DIR@" ~= "") then + pushenv("LBANN_PYTHON_VENV_DIR","@LBANN_PYTHON_VENV_DIR@") + pushenv("LBANN_PYTHON_VENV_SITE_PACKAGES","@LBANN_PYTHON_VENV_DIR@/@CMAKE_INSTALL_PYTHONDIR@") + prepend_path("PYTHONPATH","@LBANN_PYTHON_VENV_DIR@/@CMAKE_INSTALL_PYTHONDIR@") + execute {cmd="source @LBANN_PYTHON_VENV_DIR@/bin/activate", modeA={"load"}} + execute {cmd="deactivate", modeA={"unload"}} +end pushenv("LBANN_DIR","@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_DIR@") diff --git a/cmake/configure_files/lbann_module.tcl.in b/cmake/configure_files/lbann_module.tcl.in deleted file mode 100644 index cd50424b55..0000000000 --- a/cmake/configure_files/lbann_module.tcl.in +++ /dev/null @@ -1,90 +0,0 @@ -#%Module - -################################################################################ -## Copyright (c) 2014-2022, Lawrence Livermore National Security, LLC. -## Produced at the Lawrence Livermore National Laboratory. -## Written by the LBANN Research Team (B. Van Essen, et al.) listed in -## the CONTRIBUTORS file. -## -## LLNL-CODE-697807. -## All rights reserved. -## -## This file is part of LBANN: Livermore Big Artificial Neural Network -## Toolkit. For details, see http://software.llnl.gov/LBANN or -## https://github.com/LLNL/LBANN. -## -## Licensed under the Apache License, Version 2.0 (the "Licensee"); you -## may not use this file except in compliance with the License. You may -## obtain a copy of the License at: -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -## implied. See the License for the specific language governing -## permissions and limitations under the license. -################################################################################ - -# Lua (and hence LMod) should be preferred, but this will -# satisfy... less modern system needs. - -set name lbann -set version @LBANN_VERSION@ -set root @CMAKE_INSTALL_PREFIX@ - -conflict $name - -set fullname LBANN -set url https://github.com/llnl/lbann -set docs https://lbann.readthedocs.io - -set description "LBANN: Livermore Big Artificial Neural Network Toolkit." - -proc ModulesHelp { } { - global description url docs - puts stderr "Description - $description" - puts stderr - puts stderr "Docs - $url" -} - -module-whatis "Package: LBANN -Version: @LBANN_VERSION@ -Description: Livermore Big Artificial Neural Network Toolkit. - A distributed memory, HPC-optimized, model and data parallel - training toolkit for deep neural networks. -URL: https://github.com/llnl/lbann -Configuration: - CMAKE_INSTALL_PREFIX: @CMAKE_INSTALL_PREFIX@ - CMAKE_BUILD_TYPE: @CMAKE_BUILD_TYPE@ - CXX Compiler: @CMAKE_CXX_COMPILER@ - CXX FLAGS: @CMAKE_CXX_FLAGS@ - CXX FLAGS_DEBUG: @CMAKE_CXX_FLAGS_DEBUG@ - CXX FLAGS_RELWITHDEBINFO: @CMAKE_CXX_FLAGS_RELWITHDEBINFO@ - CXX FLAGS_RELEASE: @CMAKE_CXX_FLAGS_RELEASE@ - LBANN_GNU_LINUX: @LBANN_GNU_LINUX@ - LBANN_HAS_HYDROGEN: @LBANN_HAS_HYDROGEN@ - LBANN_HAS_OPENCV: @LBANN_HAS_OPENCV@ - LBANN_HAS_CEREAL: @LBANN_HAS_CEREAL@ - LBANN_HAS_CUDA: @LBANN_HAS_CUDA@ - LBANN_HAS_CUDNN: @LBANN_HAS_CUDNN@ - LBANN_HAS_NCCL2: @LBANN_HAS_NCCL2@ - LBANN_HAS_PROTOBUF: @LBANN_HAS_PROTOBUF@ - LBANN_HAS_CNPY: @LBANN_HAS_CNPY@ - LBANN_HAS_TBINF: @LBANN_HAS_TBINF@ - LBANN_HAS_VTUNE: @LBANN_HAS_VTUNE@ - LBANN_NVPROF: @LBANN_NVPROF@ - LBANN_HAS_DOXYGEN: @LBANN_HAS_DOXYGEN@ - LBANN_HAS_LBANN_PROTO: @LBANN_HAS_LBANN_PROTO@ - LBANN_HAS_ALUMINUM: @LBANN_HAS_ALUMINUM@ - LBANN_HAS_PYTHON: @LBANN_HAS_PYTHON@" - -prepend-path PATH $root/@CMAKE_INSTALL_BINDIR@ -prepend-path PYTHONPATH @PYTHON_INSTALL_PREFIX@/@CMAKE_INSTALL_PYTHONDIR@ -prepend-path LBANN_PYTHON_SITE_PACKAGES @PYTHON_INSTALL_PREFIX@/@CMAKE_INSTALL_PYTHONDIR@ - -pushenv LBANN_PYTHON_VENV_DIR @LBANN_PYTHON_VENV_DIR@ -pushenv LBANN_PYTHON_VENV_SITE_PACKAGES @LBANN_PYTHON_VENV_DIR@/@CMAKE_INSTALL_PYTHONDIR@ -prepend-path PYTHONPATH @LBANN_PYTHON_VENV_DIR@/@CMAKE_INSTALL_PYTHONDIR@ - -pushenv LBANN_DIR @CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_DIR@