From a545ccd48316422963bc14ef07d307a40e9d35a6 Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Wed, 10 Jan 2024 08:21:54 -0500 Subject: [PATCH 01/29] Update soconda.sh - Simplify conda package installation. - Remove post-installation configurations. --- soconda.sh | 68 +++--------------------------------------------------- 1 file changed, 3 insertions(+), 65 deletions(-) diff --git a/soconda.sh b/soconda.sh index befc6e0..1af211c 100755 --- a/soconda.sh +++ b/soconda.sh @@ -180,21 +180,10 @@ else fi # Install conda packages. - -conda_pkgs="" -for pkgfile in "${scriptdir}/config/common.txt" "${confdir}/packages_conda.txt"; do - while IFS='' read -r line || [[ -n "${line}" ]]; do - # Is this line commented? - comment=$(echo "${line}" | cut -c 1) - if [ "${comment}" != "#" ]; then - pkgname="${line}" - conda_pkgs="${conda_pkgs} ${pkgname}" - fi - done < "${pkgfile}" -done - echo "Installing conda packages..." | tee "log_conda" -conda install --yes ${conda_pkgs} \ +conda install --yes --file "${scriptdir}/config/common.txt" \ + | tee -a "log_conda" 2>&1 +conda install --yes --file "${confdir}/packages_conda.txt" \ | tee -a "log_conda" 2>&1 # The "cc" symlink from the compilers package shadows Cray's MPI C compiler... rm -f "${CONDA_PREFIX}/bin/cc" @@ -290,54 +279,3 @@ while IFS='' read -r line || [[ -n "${line}" ]]; do fi done < "${confdir}/packages_pip.txt" -# Create jupyter kernel launcher -kern="${CONDA_PREFIX}/bin/soconda_run_kernel.sh" -echo "#!/bin/bash" > "${kern}" -echo "conn=\$1" >> "${kern}" -echo "source \"${conda_dir}/etc/profile.d/conda.sh\"" >> "${kern}" -echo "conda activate \"${fullenv}\"" >> "${kern}" -echo "if [ -n \${NERSC_HOST} ]; then export DISABLE_MPI=true fi" >> "${kern}" -echo "exec python3 -m ipykernel -f \${conn}" >> "${kern}" -chmod +x "${kern}" - -# Create and install module file and jupyter init script - -if [ -z "${moduledir}" ]; then - # No centralized directory was specified for modulefiles. Make - # a subdirectory within the environment itself. - moduledir="${CONDA_PREFIX}/modulefiles" -fi -mkdir -p "${moduledir}/${envroot}" -if [ -z "${LMOD_VERSION}" ]; then - # Using TCL modules - input_mod="${scriptdir}/templates/modulefile_tcl.in" - outmod="${moduledir}/${envroot}/${version}" -else - # Using LMOD - input_mod="${scriptdir}/templates/modulefile_lua.in" - outmod="${moduledir}/${envroot}/${version}.lua" -fi -rm -f "${outmod}" - -confsub="-e 's#@VERSION@#${version}#g'" -confsub="${confsub} -e 's#@BASE@#${conda_dir}#g'" -confsub="${confsub} -e 's#@ENVNAME@#${fullenv}#g'" -confsub="${confsub} -e 's#@ENVPREFIX@#${CONDA_PREFIX}#g'" -confsub="${confsub} -e 's#@PYVER@#${pyver}#g'" - -while IFS='' read -r line || [[ -n "${line}" ]]; do - if [[ "${line}" =~ @MODLOAD@ ]]; then - if [ -e "${modinit}" ]; then - cat "${modinit}" >> "${outmod}" - fi - else - echo "${line}" | eval sed ${confsub} >> "${outmod}" - fi -done < "${input_mod}" - -rm -f "${out_jupyter}" -out_jupyter="${CONDA_PREFIX}/bin/soconda_jupyter.sh" -while IFS='' read -r line || [[ -n "${line}" ]]; do - echo "${line}" | eval sed ${confsub} >> "${out_jupyter}" -done < "${scriptdir}/templates/jupyter.sh.in" -chmod +x "${out_jupyter}" From d265a88e3f8ad2daf98bd21840b898e72732081b Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Wed, 10 Jan 2024 16:42:41 -0500 Subject: [PATCH 02/29] Add micromamba support to soconda.sh --- soconda.sh | 142 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 82 insertions(+), 60 deletions(-) diff --git a/soconda.sh b/soconda.sh index 1af211c..b91bd03 100755 --- a/soconda.sh +++ b/soconda.sh @@ -68,6 +68,23 @@ if [ -z "${envname}" ]; then echo "Environment root name not specified, using \"soconda\"" envname="soconda" fi +# The full environment name, including the root and version. +fullenv="${envname}_${version}" +# Determine whether the new environment is a name or a full path. +env_noslash=$(echo "${fullenv}" | sed -e 's/\///g') +if [ "${env_noslash}" != "${fullenv}" ]; then + # This was a path + is_path='yes' + # Make sure this is a full path + if [[ "${envname:0:1}" != '/' ]] ; then + # The path does not start at root + echo "Please provide a full path." + exit 1 + fi +else + is_path='no' +fi + if [ -z "${config}" ]; then echo "No config specified, using \"default\"" @@ -76,12 +93,6 @@ else echo "Using config \"${config}\"" fi -# The env root name, used for the name of the generated module file -envroot=$(basename ${envname}) - -# The full environment name, including the root and version. -fullenv="${envname}_${version}" - # The path to the selected config directory confdir="${scriptdir}/config/${config}" if [ ! -d "${confdir}" ]; then @@ -89,36 +100,48 @@ if [ ! -d "${confdir}" ]; then exit 1 fi -# The optional module init for this config -modinit="${confdir}/module_init" -# Activate the base environment if [ -n "${base}" ]; then conda_dir="${base}" + # Initialize conda + source "${conda_dir}/etc/profile.d/conda.sh" + # Conda initialization script will create conda function, + # but micromamba will create micromamba function. + # Here create a conda_exec function that unify conda and micromamba + # installation. + conda_exec () { conda "$@" ; } else # User did not specify where to find it - if [ -z "$(which conda)" ]; then - # conda is not in the path + if [ -n "$CONDA_EXE" ]; then + echo "Find conda command at ${CONDA_EXE}" + conda_dir="$(dirname $(dirname $CONDA_EXE))" + # Initialize conda + eval "$("$CONDA_EXE" 'shell.bash' 'hook')" + conda_exec () { conda "$@" ; } + elif [ -n "$MAMBA_EXE" ]; then + echo "Find micromamba command at ${MAMBA_EXE}" + conda_dir="$MAMBA_ROOT_PREFIX" + # Initialize micromamba + eval "$("$MAMBA_EXE" shell hook --shell bash)" + conda_exec () { micromamba "$@" ; } + else + # Could not find conda or micromamba echo "You must either activate the conda base environment before" echo "running this script, or you must specify the path to the base" echo "install with the \"-b \" option." exit 1 - else - # Conda is in the path - conda_dir="$(dirname $(dirname $(which conda)))" fi fi -# Make sure that conda is initialized -source "${conda_dir}/etc/profile.d/conda.sh" -conda activate base # Conda package cache. In some cases this can get really big, so # we point it to a temp location unless the user has overridden it. if [ -z ${CONDA_PKGS_DIRS} ]; then if [ -z ${NERSC_HOST} ]; then # Standard system - conda_tmp=$(mktemp -d) + # Create temp direcotry + mkdir -p "$scriptdir/tmpfs" + conda_tmp=$(mktemp -d --tmpdir="$scriptdir/tmpfs") else # Running at NERSC, use a directory in scratch conda_tmp="${SCRATCH}/tmp_soconda" @@ -127,36 +150,25 @@ if [ -z ${CONDA_PKGS_DIRS} ]; then export CONDA_PKGS_DIRS="${conda_tmp}" fi -# Determine whether the new environment is a name or a full path. -env_noslash=$(echo "${fullenv}" | sed -e 's/\///g') -is_path=no -if [ "${env_noslash}" != "${fullenv}" ]; then - # This was a path - is_path=yes - env_check="" - if [ -e "${fullenv}/bin/conda" ]; then - # It already exists - env_check="${fullenv}" - fi -else - env_check=$(conda env list | grep "${fullenv} ") -fi # Extract the python version we are using (if not the default) # and pass that to the conda create command. python_version=$(cat "${confdir}/packages_conda.txt" | grep 'python=') + +# Check this env exist or not +# env_check would be empty if not exist +env_check=$(conda_exec env list | grep "${fullenv}") if [ -z "${env_check}" ]; then - # Environment does not yet exist. Create it. if [ ${is_path} = "no" ]; then echo "Creating new environment \"${fullenv}\"" - conda create --yes -n "${fullenv}" "${python_version}" + conda_exec create --yes -n "${fullenv}" "${python_version}" else echo "Creating new environment \"${fullenv}\"" - conda create --yes -p "${fullenv}" "${python_version}" + conda_exec create --yes -p "${fullenv}" "${python_version}" fi echo "Activating environment \"${fullenv}\"" - conda activate "${fullenv}" + conda_exec activate "${fullenv}" # Create condarc for this environment echo "# condarc for soconda" > "${CONDA_PREFIX}/.condarc" @@ -168,50 +180,39 @@ if [ -z "${env_check}" ]; then echo "solver: libmamba" >> "${CONDA_PREFIX}/.condarc" # Reactivate to pick up changes - conda deactivate - conda activate "${fullenv}" + conda_exec deactivate + conda_exec activate "${fullenv}" # Copy logo files cp "${scriptdir}"/logo*.png "${CONDA_PREFIX}/" else echo "Activating environment \"${fullenv}\"" - conda activate "${fullenv}" - conda env list + conda_exec activate "${fullenv}" fi +conda_exec env list + # Install conda packages. echo "Installing conda packages..." | tee "log_conda" -conda install --yes --file "${scriptdir}/config/common.txt" \ +conda_exec install --yes --file "${scriptdir}/config/common.txt" \ | tee -a "log_conda" 2>&1 -conda install --yes --file "${confdir}/packages_conda.txt" \ +conda_exec install --yes --file "${confdir}/packages_conda.txt" \ | tee -a "log_conda" 2>&1 # The "cc" symlink from the compilers package shadows Cray's MPI C compiler... rm -f "${CONDA_PREFIX}/bin/cc" -conda deactivate -conda activate "${fullenv}" - -# Use pipgrip to install dependencies of pip packages with conda. -pip install pipgrip - -mkdir -p "${CONDA_PREFIX}/conda-bld" -conda-index "${CONDA_PREFIX}/conda-bld" -conda config \ - --file "${CONDA_PREFIX}/.condarc" \ - --add channels "file://${CONDA_PREFIX}/conda-bld" +conda_exec deactivate +conda_exec activate "${fullenv}" -# Get the python site packages version -pyver=$(python3 --version 2>&1 | awk '{print $2}' | sed -e "s#\(.*\)\.\(.*\)\..*#\1.\2#") # Install mpi4py - echo "Installing mpi4py..." | tee "log_mpi4py" if [ -z "${MPICC}" ]; then echo "The MPICC environment variable is not set. Installing mpi4py" \ | tee -a "log_mpi4py" echo "from the conda package, rather than building from source." \ | tee -a "log_mpi4py" - conda install --yes openmpi mpi4py | tee -a "log_mpi4py" 2>&1 \ + conda_exec install --yes openmpi mpi4py | tee -a "log_mpi4py" 2>&1 \ # Disable the ancient openib btl, in order to avoid a harmless warning echo 'btl = ^openib' >> "${CONDA_PREFIX}/etc/openmpi-mca-params.conf" else @@ -220,7 +221,18 @@ else | tee -a "log_mpi4py" 2>&1 fi + # Build local packages +# Here we use conda instead of conda_exec, because conda is a dependency +# of conda-build package. Therefore after activated ${fullenv}, +# conda command is availabe in both micromamba and miniforge installation. +# If you run $(which conda) command it should return $CONDA_PREFIX/bin/conda +# in both cases. +mkdir -p "${CONDA_PREFIX}/conda-bld" +conda index "${CONDA_PREFIX}/conda-bld" +conda config \ + --file "${CONDA_PREFIX}/.condarc" \ + --add channels "file://${CONDA_PREFIX}/conda-bld" while IFS='' read -r line || [[ -n "${line}" ]]; do # Is this line commented? @@ -229,20 +241,30 @@ while IFS='' read -r line || [[ -n "${line}" ]]; do pkgname="${line}" pkgrecipe="${scriptdir}/pkgs/${pkgname}" echo "Building local package '${pkgname}'" - conda-build ${pkgrecipe} | tee "log_${pkgname}" 2>&1 + conda build ${pkgrecipe} | tee "log_${pkgname}" 2>&1 echo "Installing local package '${pkgname}'" conda install --yes --use-local ${pkgname} fi done < "${confdir}/packages_local.txt" echo "Cleaning up build products" -conda-build purge +conda build purge + +# Remove buid directory +rm -r "${conda_tmp}" + +# Remove /tmp/pixell-* test files create by pixell/setup.py +find "/tmp" -type f -name 'pixell-*' -exec rm -rf {} \; + # Install pip packages. We install one package at a time # with no dependencies, so that we will intentionally # get an error. All dependency packages should be installed # through conda. +# Use pipgrip to install dependencies of pip packages with conda. +pip install pipgrip + echo "Installing pip packages..." | tee "log_pip" while IFS='' read -r line || [[ -n "${line}" ]]; do @@ -261,7 +283,7 @@ while IFS='' read -r line || [[ -n "${line}" ]]; do if [ -z "${depcheck}" ]; then # It is not already installed, try to install it with conda echo "Attempt to install conda package for dependency \"${name}\"..." | tee -a "log_pip" 2>&1 - conda install --yes ${name} | tee -a "log_pip" 2>&1 + conda_exec install --yes ${name} | tee -a "log_pip" 2>&1 if [ $? -ne 0 ]; then echo " No conda package available for dependency \"${name}\"" | tee -a "log_pip" 2>&1 echo " Assuming pip package already installed." | tee -a "log_pip" 2>&1 From 8339adc6d7453fc93541b865d567202c71a35da8 Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Wed, 10 Jan 2024 17:00:04 -0500 Subject: [PATCH 03/29] Fix missing pyver --- soconda.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/soconda.sh b/soconda.sh index 3f3da84..345071f 100755 --- a/soconda.sh +++ b/soconda.sh @@ -302,6 +302,10 @@ while IFS='' read -r line || [[ -n "${line}" ]]; do fi done < "${confdir}/packages_pip.txt" + +# Get the python site packages version +pyver=$(python3 --version 2>&1 | awk '{print $2}' | sed -e "s#\(.*\)\.\(.*\)\..*#\1.\2#") + # Subsitutions to use when parsing input templates confsub="-e 's#@VERSION@#${version}#g'" confsub="${confsub} -e 's#@BASE@#${conda_dir}#g'" From 3f69fea7ca804e8fb525818a1976c483200eaec6 Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 11 Jan 2024 09:12:51 -0500 Subject: [PATCH 04/29] Fix sotodlib download link in packages_pip.txt --- config/minimal/packages_pip.txt | 2 +- config/perlmutter/packages_pip.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/minimal/packages_pip.txt b/config/minimal/packages_pip.txt index 768920a..1b41f5f 100644 --- a/config/minimal/packages_pip.txt +++ b/config/minimal/packages_pip.txt @@ -8,7 +8,7 @@ toast==3.0.0a20 so3g # # Latest tag -https://github.com/simonsobs/sotodlib/archive/refs/master.tar.gz +https://github.com/simonsobs/sotodlib/archive/master.tar.gz # # The mapsims package has hard-coded versions of pysm3 and # pixell, which un-installs our versions of those packages. diff --git a/config/perlmutter/packages_pip.txt b/config/perlmutter/packages_pip.txt index 066af03..ae03dcd 100644 --- a/config/perlmutter/packages_pip.txt +++ b/config/perlmutter/packages_pip.txt @@ -7,7 +7,7 @@ pysm3 #so3g # # Latest tag -https://github.com/simonsobs/sotodlib/archive/refs/master.tar.gz +https://github.com/simonsobs/sotodlib/archive/master.tar.gz # # The mapsims package has hard-coded versions of pysm3 and # pixell, which un-installs our versions of those packages. From 239bba4f8f9e81021bfcf6a016e0387b925caf9d Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 11 Jan 2024 09:14:48 -0500 Subject: [PATCH 05/29] Improve cleaning up build directory. --- soconda.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/soconda.sh b/soconda.sh index 345071f..912714a 100755 --- a/soconda.sh +++ b/soconda.sh @@ -156,7 +156,6 @@ fi # and pass that to the conda create command. python_version=$(cat "${confdir}/packages_conda.txt" | grep 'python=') - # Check this env exist or not # env_check would be empty if not exist env_check=$(conda_exec env list | grep "${fullenv}") @@ -252,10 +251,10 @@ echo "Cleaning up build products" conda build purge # Remove buid directory -rm -r "${conda_tmp}" +rm -rf "${conda_tmp}" &> /dev/null # Remove /tmp/pixell-* test files create by pixell/setup.py -find "/tmp" -type f -name 'pixell-*' -exec rm -rf {} \; +find /tmp -maxdepth 1 -type f -name 'pixell-*' -exec ls {} \; # Install pip packages. We install one package at a time @@ -269,12 +268,11 @@ pip install pipgrip echo "Installing pip packages..." | tee "log_pip" while IFS='' read -r line || [[ -n "${line}" ]]; do - # Is this line commented? - comment=$(echo "${line}" | cut -c 1) - if [ "${comment}" != "#" ]; then + # If the $line start with '#' then it's a comment. + if [ "${line:0:1}" != "#" ]; then pkg="${line}" url_check=$(echo "${pkg}" | grep '/') - if [ "x${url_check}" = "x" ]; then + if [ -z "${url_check}" ]; then echo "Checking dependencies for package \"${pkg}\"" | tee -a "log_pip" 2>&1 pkgbase=$(echo ${pkg} | sed -e 's/\([[:alnum:]_\-]*\).*/\1/') for dep in $(pipgrip --pipe "${pkg}"); do From 40061a67717887a4a6e61859dac855aac9aa5fb9 Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 11 Jan 2024 09:43:12 -0500 Subject: [PATCH 06/29] Use |& to redirect error message to tee --- soconda.sh | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/soconda.sh b/soconda.sh index 912714a..c2ab944 100755 --- a/soconda.sh +++ b/soconda.sh @@ -195,9 +195,9 @@ conda_exec env list # Install conda packages. echo "Installing conda packages..." | tee "log_conda" conda_exec install --yes --file "${scriptdir}/config/common.txt" \ - | tee -a "log_conda" 2>&1 + |& tee -a "log_conda" conda_exec install --yes --file "${confdir}/packages_conda.txt" \ - | tee -a "log_conda" 2>&1 + |& tee -a "log_conda" # The "cc" symlink from the compilers package shadows Cray's MPI C compiler... rm -f "${CONDA_PREFIX}/bin/cc" @@ -212,13 +212,13 @@ if [ -z "${MPICC}" ]; then | tee -a "log_mpi4py" echo "from the conda package, rather than building from source." \ | tee -a "log_mpi4py" - conda_exec install --yes openmpi mpi4py | tee -a "log_mpi4py" 2>&1 \ + conda_exec install --yes openmpi mpi4py |& tee -a "log_mpi4py" \ # Disable the ancient openib btl, in order to avoid a harmless warning echo 'btl = ^openib' >> "${CONDA_PREFIX}/etc/openmpi-mca-params.conf" else echo "Building mpi4py with MPICC=\"${MPICC}\"" | tee -a "log_mpi4py" pip install --force-reinstall --no-cache-dir --no-binary=mpi4py mpi4py \ - | tee -a "log_mpi4py" 2>&1 + |& tee -a "log_mpi4py" fi @@ -240,10 +240,10 @@ while IFS='' read -r line || [[ -n "${line}" ]]; do if [ "${comment}" != "#" ]; then pkgname="${line}" pkgrecipe="${scriptdir}/pkgs/${pkgname}" - echo "Building local package '${pkgname}'" | tee "log_${pkgname}" 2>&1 - conda build ${pkgrecipe} | tee -a "log_${pkgname}" 2>&1 - echo "Installing local package '${pkgname}'" | tee -a "log_${pkgname}" 2>&1 - conda install --yes --use-local ${pkgname} | tee -a "log_${pkgname}" 2>&1 + echo "Building local package '${pkgname}'" |& tee "log_${pkgname}" + conda build ${pkgrecipe} |& tee -a "log_${pkgname}" + echo "Installing local package '${pkgname}'" |& tee -a "log_${pkgname}" + conda install --yes --use-local ${pkgname} |& tee -a "log_${pkgname}" fi done < "${confdir}/packages_local.txt" @@ -273,7 +273,7 @@ while IFS='' read -r line || [[ -n "${line}" ]]; do pkg="${line}" url_check=$(echo "${pkg}" | grep '/') if [ -z "${url_check}" ]; then - echo "Checking dependencies for package \"${pkg}\"" | tee -a "log_pip" 2>&1 + echo "Checking dependencies for package \"${pkg}\"" |& tee -a "log_pip" pkgbase=$(echo ${pkg} | sed -e 's/\([[:alnum:]_\-]*\).*/\1/') for dep in $(pipgrip --pipe "${pkg}"); do name=$(echo ${dep} | sed -e 's/\([[:alnum:]_\-]*\).*/\1/') @@ -281,22 +281,22 @@ while IFS='' read -r line || [[ -n "${line}" ]]; do depcheck=$(conda list ${name} | awk '{print $1}' | grep -E "^${name}\$") if [ -z "${depcheck}" ]; then # It is not already installed, try to install it with conda - echo "Attempt to install conda package for dependency \"${name}\"..." | tee -a "log_pip" 2>&1 - conda_exec install --yes ${name} | tee -a "log_pip" 2>&1 + echo "Attempt to install conda package for dependency \"${name}\"..." |& tee -a "log_pip" + conda_exec install --yes ${name} |& tee -a "log_pip" if [ $? -ne 0 ]; then - echo " No conda package available for dependency \"${name}\"" | tee -a "log_pip" 2>&1 - echo " Assuming pip package already installed." | tee -a "log_pip" 2>&1 + echo " No conda package available for dependency \"${name}\"" |& tee -a "log_pip" + echo " Assuming pip package already installed." |& tee -a "log_pip" fi else - echo " Package for dependency \"${name}\" already installed" | tee -a "log_pip" 2>&1 + echo " Package for dependency \"${name}\" already installed" |& tee -a "log_pip" fi fi done else - echo "Pip package \"${pkg}\" is a URL, skipping dependency check" | tee -a "log_pip" 2>&1 + echo "Pip package \"${pkg}\" is a URL, skipping dependency check" |& tee -a "log_pip" fi - echo "Installing package ${pkg} with --no-deps" | tee -a "log_pip" 2>&1 - python3 -m pip install --no-deps ${pkg} | tee -a "log_pip" 2>&1 + echo "Installing package ${pkg} with --no-deps" |& tee -a "log_pip" + python3 -m pip install --no-deps ${pkg} |& tee -a "log_pip" fi done < "${confdir}/packages_pip.txt" From c801c420aa22b431dd2038878eab78da14c24fe8 Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 11 Jan 2024 12:16:06 -0500 Subject: [PATCH 07/29] bump python version --- config/minimal/packages_conda.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/minimal/packages_conda.txt b/config/minimal/packages_conda.txt index 27fe859..0595a81 100644 --- a/config/minimal/packages_conda.txt +++ b/config/minimal/packages_conda.txt @@ -1,3 +1,3 @@ # Adjust the default python version -python=3.10.* +python=3.11.* # From cd08a39690fb2daa46ac9dd778dcb4d1ef2650e7 Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 11 Jan 2024 12:27:08 -0500 Subject: [PATCH 08/29] small improvements --- soconda.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/soconda.sh b/soconda.sh index c2ab944..6c504f6 100755 --- a/soconda.sh +++ b/soconda.sh @@ -268,14 +268,15 @@ pip install pipgrip echo "Installing pip packages..." | tee "log_pip" while IFS='' read -r line || [[ -n "${line}" ]]; do + # Skip if $line is empty # If the $line start with '#' then it's a comment. - if [ "${line:0:1}" != "#" ]; then + if [[ -n "${line}" && "${line:0:1}" != "#" ]]; then pkg="${line}" url_check=$(echo "${pkg}" | grep '/') if [ -z "${url_check}" ]; then echo "Checking dependencies for package \"${pkg}\"" |& tee -a "log_pip" pkgbase=$(echo ${pkg} | sed -e 's/\([[:alnum:]_\-]*\).*/\1/') - for dep in $(pipgrip --pipe "${pkg}"); do + for dep in $(pipgrip --pipe --threads 4 "${pkg}"); do name=$(echo ${dep} | sed -e 's/\([[:alnum:]_\-]*\).*/\1/') if [ "${name}" != "${pkgbase}" ]; then depcheck=$(conda list ${name} | awk '{print $1}' | grep -E "^${name}\$") @@ -283,10 +284,6 @@ while IFS='' read -r line || [[ -n "${line}" ]]; do # It is not already installed, try to install it with conda echo "Attempt to install conda package for dependency \"${name}\"..." |& tee -a "log_pip" conda_exec install --yes ${name} |& tee -a "log_pip" - if [ $? -ne 0 ]; then - echo " No conda package available for dependency \"${name}\"" |& tee -a "log_pip" - echo " Assuming pip package already installed." |& tee -a "log_pip" - fi else echo " Package for dependency \"${name}\" already installed" |& tee -a "log_pip" fi @@ -297,6 +294,7 @@ while IFS='' read -r line || [[ -n "${line}" ]]; do fi echo "Installing package ${pkg} with --no-deps" |& tee -a "log_pip" python3 -m pip install --no-deps ${pkg} |& tee -a "log_pip" + echo -e "\n\n" fi done < "${confdir}/packages_pip.txt" From 58fcdd36f31f1756d3a4dd56dc4a51b334ef237c Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 11 Jan 2024 12:28:13 -0500 Subject: [PATCH 09/29] add missing pip packages to minimal config --- config/minimal/packages_pip.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/minimal/packages_pip.txt b/config/minimal/packages_pip.txt index 1b41f5f..5653739 100644 --- a/config/minimal/packages_pip.txt +++ b/config/minimal/packages_pip.txt @@ -3,8 +3,10 @@ quaternionarray pysm3 # Uncomment here if installing wheel, and comment out # line in packages_local.txt -pysqlite3-wheels +posix-ipc toast==3.0.0a20 +pysqlite3-wheels +qpoint so3g # # Latest tag From ceb8d1a2b0ed2451ffe768404f8978f39099ad81 Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 11 Jan 2024 14:11:38 -0500 Subject: [PATCH 10/29] remove pixell temp files --- soconda.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/soconda.sh b/soconda.sh index 6c504f6..5fbd49d 100755 --- a/soconda.sh +++ b/soconda.sh @@ -254,7 +254,7 @@ conda build purge rm -rf "${conda_tmp}" &> /dev/null # Remove /tmp/pixell-* test files create by pixell/setup.py -find /tmp -maxdepth 1 -type f -name 'pixell-*' -exec ls {} \; +find /tmp -maxdepth 1 -type f -name 'pixell-*' -exec rm {} \; # Install pip packages. We install one package at a time From 9c261a9e5812a56377b702dc659e547080947866 Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 11 Jan 2024 15:27:25 -0500 Subject: [PATCH 11/29] Add some newline improve stdout readability --- soconda.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/soconda.sh b/soconda.sh index 5fbd49d..921b2e5 100755 --- a/soconda.sh +++ b/soconda.sh @@ -159,6 +159,7 @@ python_version=$(cat "${confdir}/packages_conda.txt" | grep 'python=') # Check this env exist or not # env_check would be empty if not exist env_check=$(conda_exec env list | grep "${fullenv}") +echo -e "\n\n" if [ -z "${env_check}" ]; then if [ ${is_path} = "no" ]; then echo "Creating new environment \"${fullenv}\"" @@ -193,6 +194,7 @@ conda_exec env list # Install conda packages. +echo -e "\n\n" echo "Installing conda packages..." | tee "log_conda" conda_exec install --yes --file "${scriptdir}/config/common.txt" \ |& tee -a "log_conda" @@ -206,6 +208,7 @@ conda_exec activate "${fullenv}" # Install mpi4py +echo -e "\n\n" echo "Installing mpi4py..." | tee "log_mpi4py" if [ -z "${MPICC}" ]; then echo "The MPICC environment variable is not set. Installing mpi4py" \ @@ -240,6 +243,7 @@ while IFS='' read -r line || [[ -n "${line}" ]]; do if [ "${comment}" != "#" ]; then pkgname="${line}" pkgrecipe="${scriptdir}/pkgs/${pkgname}" + echo -e "\n\n" echo "Building local package '${pkgname}'" |& tee "log_${pkgname}" conda build ${pkgrecipe} |& tee -a "log_${pkgname}" echo "Installing local package '${pkgname}'" |& tee -a "log_${pkgname}" @@ -247,6 +251,7 @@ while IFS='' read -r line || [[ -n "${line}" ]]; do fi done < "${confdir}/packages_local.txt" +echo -e "\n\n" echo "Cleaning up build products" conda build purge @@ -263,8 +268,10 @@ find /tmp -maxdepth 1 -type f -name 'pixell-*' -exec rm {} \; # through conda. # Use pipgrip to install dependencies of pip packages with conda. +echo -e "\n\n" pip install pipgrip +echo -e "\n" echo "Installing pip packages..." | tee "log_pip" while IFS='' read -r line || [[ -n "${line}" ]]; do From 57f0011c2efb8ab156896278bbf28db37b5fb75b Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 11 Jan 2024 17:20:30 -0500 Subject: [PATCH 12/29] Speed up pip packages installation Instead of run `conda list` for every dependency, store it in a variable. `conda list` takes ~1s. Some packages like toast so3g pixell have 20+ dependencies this could save quite some time. --- soconda.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/soconda.sh b/soconda.sh index 921b2e5..d6aa0c6 100755 --- a/soconda.sh +++ b/soconda.sh @@ -274,6 +274,7 @@ pip install pipgrip echo -e "\n" echo "Installing pip packages..." | tee "log_pip" +installed_pkgs="$(conda_exec list | awk '{print $1}')" while IFS='' read -r line || [[ -n "${line}" ]]; do # Skip if $line is empty # If the $line start with '#' then it's a comment. @@ -286,11 +287,12 @@ while IFS='' read -r line || [[ -n "${line}" ]]; do for dep in $(pipgrip --pipe --threads 4 "${pkg}"); do name=$(echo ${dep} | sed -e 's/\([[:alnum:]_\-]*\).*/\1/') if [ "${name}" != "${pkgbase}" ]; then - depcheck=$(conda list ${name} | awk '{print $1}' | grep -E "^${name}\$") + depcheck=$(echo "$installed_pkgs" | grep -E "^${name}\$") if [ -z "${depcheck}" ]; then # It is not already installed, try to install it with conda echo "Attempt to install conda package for dependency \"${name}\"..." |& tee -a "log_pip" conda_exec install --yes ${name} |& tee -a "log_pip" + installed_pkgs="${installed_pkgs}"$'\n'"${name}" else echo " Package for dependency \"${name}\" already installed" |& tee -a "log_pip" fi @@ -301,6 +303,7 @@ while IFS='' read -r line || [[ -n "${line}" ]]; do fi echo "Installing package ${pkg} with --no-deps" |& tee -a "log_pip" python3 -m pip install --no-deps ${pkg} |& tee -a "log_pip" + installed_pkgs="${installed_pkgs}"$'\n'"${pkg}" echo -e "\n\n" fi done < "${confdir}/packages_pip.txt" From 89971c78e55059f8a2ac8078efb6736c8f60ddd1 Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 11 Jan 2024 20:14:34 -0500 Subject: [PATCH 13/29] Specify $TMPDIR --- docs/docs/install.md | 78 +++++++++++++++++++++++++++++--------------- soconda.sh | 1 + 2 files changed, 52 insertions(+), 27 deletions(-) diff --git a/docs/docs/install.md b/docs/docs/install.md index 47ce0fd..85b9c31 100644 --- a/docs/docs/install.md +++ b/docs/docs/install.md @@ -7,7 +7,6 @@ modulefile: $> ./soconda.sh -h Usage: ./soconda.sh - [-c ] [-e ] [-b ] [-v ] @@ -55,23 +54,52 @@ be built using your system MPI compiler. ## Example: Local System -Starting from scratch, bootstrap a small conda-forge base environment in `~/conda`: +Clone soconda repo +``` +$ git clone git@github.com:simonsobs/soconda.git +$ cd soconda +``` + +Create and activate new empty environment called `soconda` +``` +$ conda create --no-default-packages -n soconda +$ conda activate soconda +``` + +Install packages to `soconda` environment +``` +(soconda) $ export MAKEFLAGS='-j 4' +(soconda) $ bash install_pkgs.sh +``` +(The `MAKEFLAGS` doesn't seem to have any effect, investigating) + +Install JupyterLab +``` +(soconda) $ conda install jupyterlab +``` + +If running on server, start jupyterlab listening on port `12345` with command +``` +(soconda) $ cd /path/to/project +(soconda) $ nohup jupyter-lab --no-browser --port=12345 &> jupyter.log & +``` + +To list current running jupyter server: +``` +(soconda) $ jupyter server list +``` + +To connect to jupyterlab running on server, run SSH tunnel on your laptop/desktop: +``` +$ ssh -N -L 12345:localhost:12345 server_ip +``` +Then you can connect to jupyterlab with link provided by command `jupyter server list`. + +To stop jupyterlab listenging on port 12345: +``` +(soconda) $ jupyter server stop 12345 +``` - $> ./tools/bootstrap_base.sh ~/conda - $> source ~/conda/etc/profile.d/conda.sh - $> conda activate base - -Create an `soconda` environment with default name and version. However, we -decide to put all the modulefiles into a central location in the root of the -base conda install: - - $> ./soconda.sh -b ~/conda -m ~/conda/modulefiles - -Now we can load the module: - - $> module use ~/conda/modulefiles - $> module avail - $> module load soconda/XXXXXX ## Example: NERSC @@ -115,21 +143,17 @@ using this python stack. ## Customizing an Environment -When running `soconda.sh`, the system configuration to use can be specified -with the `-c` option. This should be the name of the configuration subdirectory -with the "config" top-level directory. If not specified, the "default" config -is used. If you want to dramatically change the package versions / content of -an `soconda` stack, just load the existing `base` conda environment, copy one -of the configs to a new name and edit the three lists of packages -(`packages_[conda|pip|local].txt`) to exclude certain packages or add extras. -Then install it as usual. +If you want to dramatically change the package versions / content of an +`soconda` stack, just load the existing `base` conda environment and edit the +three lists of packages (`packages_[conda|pip|local].txt`) to exclude certain +packages or add extras. Then install it as usual. ## Deleting an Environment The `soconda` environments are self contained and you can delete them by removing the path or (if using a name), removing the `/envs/` directory. You can optionally delete the modulefile and the versioned pip -local directory in your home directory. +env>` directory. You can optionally delete the modulefile and the pip local +directory in your home directory. ## Advanced Details diff --git a/soconda.sh b/soconda.sh index d6aa0c6..b9d4018 100755 --- a/soconda.sh +++ b/soconda.sh @@ -143,6 +143,7 @@ if [ -z ${CONDA_PKGS_DIRS} ]; then # Create temp direcotry mkdir -p "$scriptdir/tmpfs" conda_tmp=$(mktemp -d --tmpdir="$scriptdir/tmpfs") + export TMPDIR="$conda_tmp" else # Running at NERSC, use a directory in scratch conda_tmp="${SCRATCH}/tmp_soconda" From 5feebdc9ea932fc3f01dee2385e4cadb2cdbc09b Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 11 Jan 2024 20:50:33 -0500 Subject: [PATCH 14/29] update docs --- docs/docs/install.md | 79 +++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/docs/docs/install.md b/docs/docs/install.md index 85b9c31..dd8d8e0 100644 --- a/docs/docs/install.md +++ b/docs/docs/install.md @@ -7,6 +7,7 @@ modulefile: $> ./soconda.sh -h Usage: ./soconda.sh + [-c ] [-e ] [-b ] [-v ] @@ -24,22 +25,19 @@ clutter. ## Base Conda Environment -If you already have a conda-forge base environment, then you can skip this +If you already have a conda-forge or micromamba base environment, then you can skip this step. However, you should consider setting the "solver" in the base environment -to use libmamba. This will greatly speed up the dependency resolution -calculation. Once you decide on the install prefix for your overall conda -environment you can use the included bootstrap script. For this example, we -will use `/opt/conda` as the path to the conda base installation. Now run the -bootstrap script: +to use `libmamba`. To switch to new solver see +[this]((https://www.anaconda.com/blog/a-faster-conda-for-a-growing-community) article. +This will greatly speed up the dependency resolution +calculation. - $> ./tools/bootstrap_base "/opt/conda" - -This bootstrap will install a base system with the conda-forge channel set to -the default and using the libmamba solver. You can now source the conda -initialization file and activate this base environment: - - $> source /opt/conda/etc/profile.d/conda.sh - $> conda activate base +For new installation run following command to install miniforge +``` +curl -L "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" | bash +``` +It will set conda-forge as default channel and use `libmamba` as default solver. +After the installation you need to re-login or start a new terminal to initialize conda. After installing an `soconda` environment below, you will not need this step since it is done by the generated modulefile. @@ -53,54 +51,55 @@ compiler before running `soconda.sh`. That will cause the mpi4py package to be built using your system MPI compiler. ## Example: Local System +This installation could install `soconda` to your local computer and any cluster. Clone soconda repo ``` -$ git clone git@github.com:simonsobs/soconda.git -$ cd soconda +git clone git@github.com:simonsobs/soconda.git +cd soconda ``` -Create and activate new empty environment called `soconda` +Run the `soconda.sh` script ``` -$ conda create --no-default-packages -n soconda -$ conda activate soconda +export MAKEFLAGS='-j 4' +bash soconda.sh -e soconda -c default ``` +This will create a new environment `soconda_xxx.x.x` with version number as suffix +using `default` configuration. [More details on configuration.](## Customizing an Environment) +(The `MAKEFLAGS` doesn't seem to have any effect.) -Install packages to `soconda` environment +You could find out the name of new created environment with ``` -(soconda) $ export MAKEFLAGS='-j 4' -(soconda) $ bash install_pkgs.sh +conda env list ``` -(The `MAKEFLAGS` doesn't seem to have any effect, investigating) -Install JupyterLab +Then you can now activate the environment with ``` -(soconda) $ conda install jupyterlab +conda activate soconda_xxx.x.x ``` If running on server, start jupyterlab listening on port `12345` with command ``` -(soconda) $ cd /path/to/project -(soconda) $ nohup jupyter-lab --no-browser --port=12345 &> jupyter.log & +cd /path/to/project +nohup jupyter-lab --no-browser --port=12345 &> jupyter.log & ``` To list current running jupyter server: ``` -(soconda) $ jupyter server list +jupyter server list ``` -To connect to jupyterlab running on server, run SSH tunnel on your laptop/desktop: +To connect to jupyterlab running on server, start SSH tunnel from your laptop/desktop: ``` -$ ssh -N -L 12345:localhost:12345 server_ip +ssh -N -L 12345:localhost:12345 server_domain_or_ip ``` Then you can connect to jupyterlab with link provided by command `jupyter server list`. To stop jupyterlab listenging on port 12345: ``` -(soconda) $ jupyter server stop 12345 +jupyter server stop 12345 ``` - ## Example: NERSC At NERSC, the default provided python is from Anaconda, and does not work well @@ -143,17 +142,21 @@ using this python stack. ## Customizing an Environment -If you want to dramatically change the package versions / content of an -`soconda` stack, just load the existing `base` conda environment and edit the -three lists of packages (`packages_[conda|pip|local].txt`) to exclude certain -packages or add extras. Then install it as usual. +When running `soconda.sh`, the system configuration to use can be specified +with the `-c` option. This should be the name of the configuration subdirectory +with the "config" top-level directory. If not specified, the "default" config +is used. If you want to dramatically change the package versions / content of +an `soconda` stack, just load the existing `base` conda environment, copy one +of the configs to a new name and edit the three lists of packages +(`packages_[conda|pip|local].txt`) to exclude certain packages or add extras. +Then install it as usual. ## Deleting an Environment The `soconda` environments are self contained and you can delete them by removing the path or (if using a name), removing the `/envs/` directory. You can optionally delete the modulefile and the pip local -directory in your home directory. +env>` directory. You can optionally delete the modulefile and the versioned pip +local directory in your home directory. ## Advanced Details From 9658f76a840d2fad9ee13c4ca2cde0dc7f9c52cf Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 11 Jan 2024 20:54:50 -0500 Subject: [PATCH 15/29] update docs --- docs/docs/install.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/docs/install.md b/docs/docs/install.md index dd8d8e0..911c521 100644 --- a/docs/docs/install.md +++ b/docs/docs/install.md @@ -27,14 +27,15 @@ clutter. If you already have a conda-forge or micromamba base environment, then you can skip this step. However, you should consider setting the "solver" in the base environment -to use `libmamba`. To switch to new solver see +to use `libmamba`. To use `libmamba` solver see [this]((https://www.anaconda.com/blog/a-faster-conda-for-a-growing-community) article. This will greatly speed up the dependency resolution calculation. For new installation run following command to install miniforge ``` -curl -L "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" | bash +curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" +bash Miniforge3-$(uname)-$(uname -m).sh ``` It will set conda-forge as default channel and use `libmamba` as default solver. After the installation you need to re-login or start a new terminal to initialize conda. From 92a2d52af0673f9413387174c1351b0e2ff4655c Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 11 Jan 2024 21:03:59 -0500 Subject: [PATCH 16/29] update docs: fix links --- docs/docs/install.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/install.md b/docs/docs/install.md index 911c521..b447e7f 100644 --- a/docs/docs/install.md +++ b/docs/docs/install.md @@ -28,7 +28,7 @@ clutter. If you already have a conda-forge or micromamba base environment, then you can skip this step. However, you should consider setting the "solver" in the base environment to use `libmamba`. To use `libmamba` solver see -[this]((https://www.anaconda.com/blog/a-faster-conda-for-a-growing-community) article. +[this](https://www.anaconda.com/blog/a-faster-conda-for-a-growing-community) article. This will greatly speed up the dependency resolution calculation. @@ -66,7 +66,7 @@ export MAKEFLAGS='-j 4' bash soconda.sh -e soconda -c default ``` This will create a new environment `soconda_xxx.x.x` with version number as suffix -using `default` configuration. [More details on configuration.](## Customizing an Environment) +using `default` configuration. [More details on configuration.](#customizing-an-environment) (The `MAKEFLAGS` doesn't seem to have any effect.) You could find out the name of new created environment with From 2f6d80c74b9dec64cb3daa27874475c54d9b10cd Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Fri, 12 Jan 2024 08:16:17 -0500 Subject: [PATCH 17/29] Update docs: add qt-wayland --- docs/docs/install.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/docs/install.md b/docs/docs/install.md index b447e7f..5f0e2c5 100644 --- a/docs/docs/install.md +++ b/docs/docs/install.md @@ -79,6 +79,11 @@ Then you can now activate the environment with conda activate soconda_xxx.x.x ``` +If running on a Linux desktop that uses wayland, you also need to install the `qt-wayland` package +``` +conda install qt-wayland +``` + If running on server, start jupyterlab listening on port `12345` with command ``` cd /path/to/project From bbc680659714f84df1d6767cadebba1835f15ee0 Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Fri, 12 Jan 2024 13:20:40 -0500 Subject: [PATCH 18/29] Update docs --- docs/docs/install.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/docs/install.md b/docs/docs/install.md index 5f0e2c5..d133bc0 100644 --- a/docs/docs/install.md +++ b/docs/docs/install.md @@ -160,8 +160,9 @@ Then install it as usual. ## Deleting an Environment The `soconda` environments are self contained and you can delete them by -removing the path or (if using a name), removing the `/envs/` directory. You can optionally delete the modulefile and the versioned pip +running command `conda remove --name envname --all` or `conda remove -p /base_dir/envs/name --all`. +Or directly removing the `/envs/` directory. +You can optionally delete the modulefile and the versioned pip local directory in your home directory. ## Advanced Details From b4b95b662723e3d4256156fbbeff13cd97f206fb Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Sun, 21 Jan 2024 22:04:24 -0500 Subject: [PATCH 19/29] pkgs/toast: Use git tag as version. --- pkgs/toast/meta.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/toast/meta.yaml b/pkgs/toast/meta.yaml index 2fc5c1a..39a903b 100644 --- a/pkgs/toast/meta.yaml +++ b/pkgs/toast/meta.yaml @@ -1,6 +1,6 @@ -{% set version = "3980cbb579de56a379df000df76e69a1c1ac73ae" %} -{% set sha256 = "e2713145f758631cd6bb24b89a87f2522322a72a29400f8ba51d5144e9afe54d" %} +{% set version = "3.0.0a20" %} +{% set sha256 = "4d5ad584bf0d2e851009beb17a4ec96677b36b9c9e18893910014581bc22bb4c" %} {% set build = 0 %} @@ -9,7 +9,7 @@ package: version: {{ version }} source: - url: https://github.com/hpc4cmb/toast/archive/{{ version }}.tar.gz + url: https://github.com/hpc4cmb/toast/archive/refs/tags/{{ version }}.tar.gz sha256: {{ sha256 }} # patches: # - debug_lapack.patch From 93cc954d5bf6992501d50b1cede6edc5835883cd Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Mon, 22 Jan 2024 10:26:10 -0500 Subject: [PATCH 20/29] run_test.sh change MPI nproc to 2 Running toast test with `-np 4` will hang up during `toast.tests.io_hdf5.IoHdf5Test`. Reducing to `-np 2` solves it. --- run_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 3dd7c71..55ed4b4 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,10 +1,10 @@ #!/bin/bash # Pixell Tests -OMP_NUM_THREADS=4 test_pixell.py +#OMP_NUM_THREADS=4 test_pixell.py # Serial TOAST Tests OMP_NUM_THREADS=4 MPI_DISABLED=1 python -c 'import toast.tests; toast.tests.run()' # MPI-enabled tests -OMP_NUM_THREADS=2 mpirun -np 4 python -c 'import toast.tests; toast.tests.run()' +OMP_NUM_THREADS=2 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()' From 6b80fabb50c8c2a92684fc748c5d5885c4d0823e Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Mon, 22 Jan 2024 10:47:29 -0500 Subject: [PATCH 21/29] pkgs/so3g update sha256sum For some reason sha256sum changed. --- pkgs/so3g/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/so3g/meta.yaml b/pkgs/so3g/meta.yaml index 15aa6f2..796d128 100644 --- a/pkgs/so3g/meta.yaml +++ b/pkgs/so3g/meta.yaml @@ -1,5 +1,5 @@ {% set version = "0.1.10" %} -{% set sha256 = "4cabc243acf6a18bb8b63572f468b303c3bc4cb4e0e901e3fbe222a8ff094e11" %} +{% set sha256 = "5367cdd24ac7eff70b482ede2c32df8e1acd16e582a24e489467ec2c4760c31b" %} {% set build = 0 %} From fcf75a294873efd0e0ea5ea0e767c68308fd34f2 Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Mon, 22 Jan 2024 11:47:24 -0500 Subject: [PATCH 22/29] config/common.txt add conda-verify package. --- config/common.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/config/common.txt b/config/common.txt index 9d80cc2..f601f9f 100644 --- a/config/common.txt +++ b/config/common.txt @@ -5,6 +5,7 @@ # and parse pip dependencies with pipgrip. # conda-build +conda-verify compilers setuptools pip From bf163cb08ee1cc7e4e8a343d1573a1652571442d Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Mon, 22 Jan 2024 13:34:22 -0500 Subject: [PATCH 23/29] Add base and base-bin configuration, update corresponding documentation. The base and base-bin are for local setup, where module files and jupyter kernel script are unnecessary. --- config/base-bin/packages_conda.txt | 7 +++++++ config/base-bin/packages_local.txt | 13 +++++++++++++ config/base-bin/packages_pip.txt | 17 +++++++++++++++++ config/base-bin/post_install.sh | 15 +++++++++++++++ config/base/packages_conda.txt | 7 +++++++ config/base/packages_local.txt | 11 +++++++++++ config/base/packages_pip.txt | 14 ++++++++++++++ config/base/post_install.sh | 15 +++++++++++++++ docs/docs/install.md | 15 ++++++++++++--- 9 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 config/base-bin/packages_conda.txt create mode 100644 config/base-bin/packages_local.txt create mode 100644 config/base-bin/packages_pip.txt create mode 100644 config/base-bin/post_install.sh create mode 100644 config/base/packages_conda.txt create mode 100644 config/base/packages_local.txt create mode 100644 config/base/packages_pip.txt create mode 100644 config/base/post_install.sh diff --git a/config/base-bin/packages_conda.txt b/config/base-bin/packages_conda.txt new file mode 100644 index 0000000..2a8b83d --- /dev/null +++ b/config/base-bin/packages_conda.txt @@ -0,0 +1,7 @@ +# Adjust the default python version +python=3.11.* +# +jupyterlab +wurlitzer +plotly +plotly-resampler diff --git a/config/base-bin/packages_local.txt b/config/base-bin/packages_local.txt new file mode 100644 index 0000000..364899e --- /dev/null +++ b/config/base-bin/packages_local.txt @@ -0,0 +1,13 @@ +# This config skips all the ACT tools and just installs +# binary packages for everything. +#qpoint +#pixell +#libactpol_deps +#libactpol +#moby2 +# Remove next line once toast-3.0 is on conda-forge, +# and move to packages_conda.txt +#toast +# Uncomment next line to build so3g from source, and +# comment out line in packages_pip.txt. +#so3g diff --git a/config/base-bin/packages_pip.txt b/config/base-bin/packages_pip.txt new file mode 100644 index 0000000..5653739 --- /dev/null +++ b/config/base-bin/packages_pip.txt @@ -0,0 +1,17 @@ +pixell +quaternionarray +pysm3 +# Uncomment here if installing wheel, and comment out +# line in packages_local.txt +posix-ipc +toast==3.0.0a20 +pysqlite3-wheels +qpoint +so3g +# +# Latest tag +https://github.com/simonsobs/sotodlib/archive/master.tar.gz +# +# The mapsims package has hard-coded versions of pysm3 and +# pixell, which un-installs our versions of those packages. +#https://github.com/galsci/mapsims/archive/main.tar.gz diff --git a/config/base-bin/post_install.sh b/config/base-bin/post_install.sh new file mode 100644 index 0000000..f640d65 --- /dev/null +++ b/config/base-bin/post_install.sh @@ -0,0 +1,15 @@ +# This script is sourced in the main `soconda.sh` script. +# Any commands placed here will be executed within that shell +# and have access to all environment variables defined there. +# There are 2 variables defined here which control the optional +# creation of a modulefile to load the environment and also +# which create a small script that installs a jupyter kernel +# for the environment into a user's home directory. + +# Install a module file? +install_module=no + +# Install jupyter kernel setup script? +install_jupyter_setup=no + +# Add any other shell commands here for this system... diff --git a/config/base/packages_conda.txt b/config/base/packages_conda.txt new file mode 100644 index 0000000..2a8b83d --- /dev/null +++ b/config/base/packages_conda.txt @@ -0,0 +1,7 @@ +# Adjust the default python version +python=3.11.* +# +jupyterlab +wurlitzer +plotly +plotly-resampler diff --git a/config/base/packages_local.txt b/config/base/packages_local.txt new file mode 100644 index 0000000..352fd00 --- /dev/null +++ b/config/base/packages_local.txt @@ -0,0 +1,11 @@ +qpoint +pixell +libactpol_deps +libactpol +moby2 +# Remove next line once toast-3.0 is on conda-forge, +# and move to packages_conda.txt +toast +# Uncomment next line to build so3g from source, and +# comment out line in packages_pip.txt. +so3g diff --git a/config/base/packages_pip.txt b/config/base/packages_pip.txt new file mode 100644 index 0000000..ae03dcd --- /dev/null +++ b/config/base/packages_pip.txt @@ -0,0 +1,14 @@ +quaternionarray +pysm3 +# +# Uncomment here if installing wheel, and comment out +# line in packages_local.txt +#pysqlite3-wheels +#so3g +# +# Latest tag +https://github.com/simonsobs/sotodlib/archive/master.tar.gz +# +# The mapsims package has hard-coded versions of pysm3 and +# pixell, which un-installs our versions of those packages. +#https://github.com/galsci/mapsims/archive/main.tar.gz diff --git a/config/base/post_install.sh b/config/base/post_install.sh new file mode 100644 index 0000000..f640d65 --- /dev/null +++ b/config/base/post_install.sh @@ -0,0 +1,15 @@ +# This script is sourced in the main `soconda.sh` script. +# Any commands placed here will be executed within that shell +# and have access to all environment variables defined there. +# There are 2 variables defined here which control the optional +# creation of a modulefile to load the environment and also +# which create a small script that installs a jupyter kernel +# for the environment into a user's home directory. + +# Install a module file? +install_module=no + +# Install jupyter kernel setup script? +install_jupyter_setup=no + +# Add any other shell commands here for this system... diff --git a/docs/docs/install.md b/docs/docs/install.md index d133bc0..7cd7601 100644 --- a/docs/docs/install.md +++ b/docs/docs/install.md @@ -52,7 +52,7 @@ compiler before running `soconda.sh`. That will cause the mpi4py package to be built using your system MPI compiler. ## Example: Local System -This installation could install `soconda` to your local computer and any cluster. +This installation could install `soconda` to your local computer and any remote server or cluster. Clone soconda repo ``` @@ -63,10 +63,12 @@ cd soconda Run the `soconda.sh` script ``` export MAKEFLAGS='-j 4' -bash soconda.sh -e soconda -c default +bash soconda.sh -e soconda -c base ``` This will create a new environment `soconda_xxx.x.x` with version number as suffix -using `default` configuration. [More details on configuration.](#customizing-an-environment) +using `base` configuration. [More details on configuration.](#customizing-an-environment) +If you don't want to compile some packages like `toast` `so3g`, you can use `base-bin` configuration instead. +This will install binary packages with `pip`, but try to install as many packages with `conda` as possible. (The `MAKEFLAGS` doesn't seem to have any effect.) You could find out the name of new created environment with @@ -157,6 +159,13 @@ of the configs to a new name and edit the three lists of packages (`packages_[conda|pip|local].txt`) to exclude certain packages or add extras. Then install it as usual. +| Configuration | Description | +| --- | --- | +| default | Default/reference configuration. It will also set up module file (require [Modules](https://docs.fedoraproject.org/en-US/modularity/using-modules/)) and jupyter kernel script. | +| base | Same as `default` but without module file and jupyter kernel script. | +| base-bin | Same as `base` but will not compile [packages](/pkgs). It will install `pip` packages instead. | +| minimal | Minimal installation (without jupyterlab etc.) And it will not compile [packages](/pkgs). | + ## Deleting an Environment The `soconda` environments are self contained and you can delete them by From 5b712186be0f03b4605a557e702fadbafd654431 Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 25 Jan 2024 08:54:15 -0500 Subject: [PATCH 24/29] Revert "Add base and base-bin configuration, update corresponding documentation." This reverts commit bf163cb08ee1cc7e4e8a343d1573a1652571442d. --- config/base-bin/packages_conda.txt | 7 ------- config/base-bin/packages_local.txt | 13 ------------- config/base-bin/packages_pip.txt | 17 ----------------- config/base-bin/post_install.sh | 15 --------------- config/base/packages_conda.txt | 7 ------- config/base/packages_local.txt | 11 ----------- config/base/packages_pip.txt | 14 -------------- config/base/post_install.sh | 15 --------------- docs/docs/install.md | 15 +++------------ 9 files changed, 3 insertions(+), 111 deletions(-) delete mode 100644 config/base-bin/packages_conda.txt delete mode 100644 config/base-bin/packages_local.txt delete mode 100644 config/base-bin/packages_pip.txt delete mode 100644 config/base-bin/post_install.sh delete mode 100644 config/base/packages_conda.txt delete mode 100644 config/base/packages_local.txt delete mode 100644 config/base/packages_pip.txt delete mode 100644 config/base/post_install.sh diff --git a/config/base-bin/packages_conda.txt b/config/base-bin/packages_conda.txt deleted file mode 100644 index 2a8b83d..0000000 --- a/config/base-bin/packages_conda.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Adjust the default python version -python=3.11.* -# -jupyterlab -wurlitzer -plotly -plotly-resampler diff --git a/config/base-bin/packages_local.txt b/config/base-bin/packages_local.txt deleted file mode 100644 index 364899e..0000000 --- a/config/base-bin/packages_local.txt +++ /dev/null @@ -1,13 +0,0 @@ -# This config skips all the ACT tools and just installs -# binary packages for everything. -#qpoint -#pixell -#libactpol_deps -#libactpol -#moby2 -# Remove next line once toast-3.0 is on conda-forge, -# and move to packages_conda.txt -#toast -# Uncomment next line to build so3g from source, and -# comment out line in packages_pip.txt. -#so3g diff --git a/config/base-bin/packages_pip.txt b/config/base-bin/packages_pip.txt deleted file mode 100644 index 5653739..0000000 --- a/config/base-bin/packages_pip.txt +++ /dev/null @@ -1,17 +0,0 @@ -pixell -quaternionarray -pysm3 -# Uncomment here if installing wheel, and comment out -# line in packages_local.txt -posix-ipc -toast==3.0.0a20 -pysqlite3-wheels -qpoint -so3g -# -# Latest tag -https://github.com/simonsobs/sotodlib/archive/master.tar.gz -# -# The mapsims package has hard-coded versions of pysm3 and -# pixell, which un-installs our versions of those packages. -#https://github.com/galsci/mapsims/archive/main.tar.gz diff --git a/config/base-bin/post_install.sh b/config/base-bin/post_install.sh deleted file mode 100644 index f640d65..0000000 --- a/config/base-bin/post_install.sh +++ /dev/null @@ -1,15 +0,0 @@ -# This script is sourced in the main `soconda.sh` script. -# Any commands placed here will be executed within that shell -# and have access to all environment variables defined there. -# There are 2 variables defined here which control the optional -# creation of a modulefile to load the environment and also -# which create a small script that installs a jupyter kernel -# for the environment into a user's home directory. - -# Install a module file? -install_module=no - -# Install jupyter kernel setup script? -install_jupyter_setup=no - -# Add any other shell commands here for this system... diff --git a/config/base/packages_conda.txt b/config/base/packages_conda.txt deleted file mode 100644 index 2a8b83d..0000000 --- a/config/base/packages_conda.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Adjust the default python version -python=3.11.* -# -jupyterlab -wurlitzer -plotly -plotly-resampler diff --git a/config/base/packages_local.txt b/config/base/packages_local.txt deleted file mode 100644 index 352fd00..0000000 --- a/config/base/packages_local.txt +++ /dev/null @@ -1,11 +0,0 @@ -qpoint -pixell -libactpol_deps -libactpol -moby2 -# Remove next line once toast-3.0 is on conda-forge, -# and move to packages_conda.txt -toast -# Uncomment next line to build so3g from source, and -# comment out line in packages_pip.txt. -so3g diff --git a/config/base/packages_pip.txt b/config/base/packages_pip.txt deleted file mode 100644 index ae03dcd..0000000 --- a/config/base/packages_pip.txt +++ /dev/null @@ -1,14 +0,0 @@ -quaternionarray -pysm3 -# -# Uncomment here if installing wheel, and comment out -# line in packages_local.txt -#pysqlite3-wheels -#so3g -# -# Latest tag -https://github.com/simonsobs/sotodlib/archive/master.tar.gz -# -# The mapsims package has hard-coded versions of pysm3 and -# pixell, which un-installs our versions of those packages. -#https://github.com/galsci/mapsims/archive/main.tar.gz diff --git a/config/base/post_install.sh b/config/base/post_install.sh deleted file mode 100644 index f640d65..0000000 --- a/config/base/post_install.sh +++ /dev/null @@ -1,15 +0,0 @@ -# This script is sourced in the main `soconda.sh` script. -# Any commands placed here will be executed within that shell -# and have access to all environment variables defined there. -# There are 2 variables defined here which control the optional -# creation of a modulefile to load the environment and also -# which create a small script that installs a jupyter kernel -# for the environment into a user's home directory. - -# Install a module file? -install_module=no - -# Install jupyter kernel setup script? -install_jupyter_setup=no - -# Add any other shell commands here for this system... diff --git a/docs/docs/install.md b/docs/docs/install.md index 7cd7601..d133bc0 100644 --- a/docs/docs/install.md +++ b/docs/docs/install.md @@ -52,7 +52,7 @@ compiler before running `soconda.sh`. That will cause the mpi4py package to be built using your system MPI compiler. ## Example: Local System -This installation could install `soconda` to your local computer and any remote server or cluster. +This installation could install `soconda` to your local computer and any cluster. Clone soconda repo ``` @@ -63,12 +63,10 @@ cd soconda Run the `soconda.sh` script ``` export MAKEFLAGS='-j 4' -bash soconda.sh -e soconda -c base +bash soconda.sh -e soconda -c default ``` This will create a new environment `soconda_xxx.x.x` with version number as suffix -using `base` configuration. [More details on configuration.](#customizing-an-environment) -If you don't want to compile some packages like `toast` `so3g`, you can use `base-bin` configuration instead. -This will install binary packages with `pip`, but try to install as many packages with `conda` as possible. +using `default` configuration. [More details on configuration.](#customizing-an-environment) (The `MAKEFLAGS` doesn't seem to have any effect.) You could find out the name of new created environment with @@ -159,13 +157,6 @@ of the configs to a new name and edit the three lists of packages (`packages_[conda|pip|local].txt`) to exclude certain packages or add extras. Then install it as usual. -| Configuration | Description | -| --- | --- | -| default | Default/reference configuration. It will also set up module file (require [Modules](https://docs.fedoraproject.org/en-US/modularity/using-modules/)) and jupyter kernel script. | -| base | Same as `default` but without module file and jupyter kernel script. | -| base-bin | Same as `base` but will not compile [packages](/pkgs). It will install `pip` packages instead. | -| minimal | Minimal installation (without jupyterlab etc.) And it will not compile [packages](/pkgs). | - ## Deleting an Environment The `soconda` environments are self contained and you can delete them by From 7ed6bd5a557c1864960fd12b4e10bcd2cc86e6d7 Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 25 Jan 2024 09:26:05 -0500 Subject: [PATCH 25/29] Update config Add binary and site, update miniaml --- config/binary/packages_conda.txt | 7 +++++++ config/binary/packages_local.txt | 0 config/binary/packages_pip.txt | 17 +++++++++++++++++ config/binary/post_install.sh | 15 +++++++++++++++ config/minimal/packages_local.txt | 13 ------------- config/minimal/post_install.sh | 4 ++-- config/site/packages_conda.txt | 6 ++++++ config/site/packages_local.txt | 11 +++++++++++ config/site/packages_pip.txt | 14 ++++++++++++++ config/site/post_install.sh | 15 +++++++++++++++ 10 files changed, 87 insertions(+), 15 deletions(-) create mode 100644 config/binary/packages_conda.txt create mode 100644 config/binary/packages_local.txt create mode 100644 config/binary/packages_pip.txt create mode 100644 config/binary/post_install.sh create mode 100644 config/site/packages_conda.txt create mode 100644 config/site/packages_local.txt create mode 100644 config/site/packages_pip.txt create mode 100644 config/site/post_install.sh diff --git a/config/binary/packages_conda.txt b/config/binary/packages_conda.txt new file mode 100644 index 0000000..48fc35e --- /dev/null +++ b/config/binary/packages_conda.txt @@ -0,0 +1,7 @@ +# Adjust the default python version +python=3.10.* +# +jupyterlab +wurlitzer +plotly +plotly-resampler diff --git a/config/binary/packages_local.txt b/config/binary/packages_local.txt new file mode 100644 index 0000000..e69de29 diff --git a/config/binary/packages_pip.txt b/config/binary/packages_pip.txt new file mode 100644 index 0000000..5653739 --- /dev/null +++ b/config/binary/packages_pip.txt @@ -0,0 +1,17 @@ +pixell +quaternionarray +pysm3 +# Uncomment here if installing wheel, and comment out +# line in packages_local.txt +posix-ipc +toast==3.0.0a20 +pysqlite3-wheels +qpoint +so3g +# +# Latest tag +https://github.com/simonsobs/sotodlib/archive/master.tar.gz +# +# The mapsims package has hard-coded versions of pysm3 and +# pixell, which un-installs our versions of those packages. +#https://github.com/galsci/mapsims/archive/main.tar.gz diff --git a/config/binary/post_install.sh b/config/binary/post_install.sh new file mode 100644 index 0000000..5b3bcce --- /dev/null +++ b/config/binary/post_install.sh @@ -0,0 +1,15 @@ +# This script is sourced in the main `soconda.sh` script. +# Any commands placed here will be executed within that shell +# and have access to all environment variables defined there. +# There are 2 variables defined here which control the optional +# creation of a modulefile to load the environment and also +# which create a small script that installs a jupyter kernel +# for the environment into a user's home directory. + +# Install a module file? +install_module=yes + +# Install jupyter kernel setup script? +install_jupyter_setup=yes + +# Add any other shell commands here for this system... diff --git a/config/minimal/packages_local.txt b/config/minimal/packages_local.txt index 364899e..e69de29 100644 --- a/config/minimal/packages_local.txt +++ b/config/minimal/packages_local.txt @@ -1,13 +0,0 @@ -# This config skips all the ACT tools and just installs -# binary packages for everything. -#qpoint -#pixell -#libactpol_deps -#libactpol -#moby2 -# Remove next line once toast-3.0 is on conda-forge, -# and move to packages_conda.txt -#toast -# Uncomment next line to build so3g from source, and -# comment out line in packages_pip.txt. -#so3g diff --git a/config/minimal/post_install.sh b/config/minimal/post_install.sh index f640d65..5b3bcce 100644 --- a/config/minimal/post_install.sh +++ b/config/minimal/post_install.sh @@ -7,9 +7,9 @@ # for the environment into a user's home directory. # Install a module file? -install_module=no +install_module=yes # Install jupyter kernel setup script? -install_jupyter_setup=no +install_jupyter_setup=yes # Add any other shell commands here for this system... diff --git a/config/site/packages_conda.txt b/config/site/packages_conda.txt new file mode 100644 index 0000000..45632ed --- /dev/null +++ b/config/site/packages_conda.txt @@ -0,0 +1,6 @@ +# Adjust the default python version +python=3.10.* +# +wurlitzer +plotly +plotly-resampler diff --git a/config/site/packages_local.txt b/config/site/packages_local.txt new file mode 100644 index 0000000..352fd00 --- /dev/null +++ b/config/site/packages_local.txt @@ -0,0 +1,11 @@ +qpoint +pixell +libactpol_deps +libactpol +moby2 +# Remove next line once toast-3.0 is on conda-forge, +# and move to packages_conda.txt +toast +# Uncomment next line to build so3g from source, and +# comment out line in packages_pip.txt. +so3g diff --git a/config/site/packages_pip.txt b/config/site/packages_pip.txt new file mode 100644 index 0000000..ae03dcd --- /dev/null +++ b/config/site/packages_pip.txt @@ -0,0 +1,14 @@ +quaternionarray +pysm3 +# +# Uncomment here if installing wheel, and comment out +# line in packages_local.txt +#pysqlite3-wheels +#so3g +# +# Latest tag +https://github.com/simonsobs/sotodlib/archive/master.tar.gz +# +# The mapsims package has hard-coded versions of pysm3 and +# pixell, which un-installs our versions of those packages. +#https://github.com/galsci/mapsims/archive/main.tar.gz diff --git a/config/site/post_install.sh b/config/site/post_install.sh new file mode 100644 index 0000000..5b3bcce --- /dev/null +++ b/config/site/post_install.sh @@ -0,0 +1,15 @@ +# This script is sourced in the main `soconda.sh` script. +# Any commands placed here will be executed within that shell +# and have access to all environment variables defined there. +# There are 2 variables defined here which control the optional +# creation of a modulefile to load the environment and also +# which create a small script that installs a jupyter kernel +# for the environment into a user's home directory. + +# Install a module file? +install_module=yes + +# Install jupyter kernel setup script? +install_jupyter_setup=yes + +# Add any other shell commands here for this system... From 5a0bef3a4218a484bd1a2913d56add7c442c8cc5 Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 25 Jan 2024 09:47:34 -0500 Subject: [PATCH 26/29] Add empty module_init to site config --- config/site/module_init | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/site/module_init diff --git a/config/site/module_init b/config/site/module_init new file mode 100644 index 0000000..c4cd8c8 --- /dev/null +++ b/config/site/module_init @@ -0,0 +1 @@ +-- No module dependencies From ec5418737ab025b891fd8d10dcbdbff3fda72c7f Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 25 Jan 2024 10:00:36 -0500 Subject: [PATCH 27/29] Update install.md change to bootstrap_base script --- docs/docs/install.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/docs/install.md b/docs/docs/install.md index d133bc0..4c7efa9 100644 --- a/docs/docs/install.md +++ b/docs/docs/install.md @@ -32,11 +32,11 @@ to use `libmamba`. To use `libmamba` solver see This will greatly speed up the dependency resolution calculation. -For new installation run following command to install miniforge +For new installation run following script to install miniforge ``` -curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" -bash Miniforge3-$(uname)-$(uname -m).sh +./tools/bootstrap_base "$HOME/miniforge3" ``` +This will intall conda to `$HOME/miniforge3` directory. It will set conda-forge as default channel and use `libmamba` as default solver. After the installation you need to re-login or start a new terminal to initialize conda. @@ -68,6 +68,7 @@ bash soconda.sh -e soconda -c default This will create a new environment `soconda_xxx.x.x` with version number as suffix using `default` configuration. [More details on configuration.](#customizing-an-environment) (The `MAKEFLAGS` doesn't seem to have any effect.) +If you want to specify a conda base directory add `-b "$HOME/miniforge3"` argument to `soconda.sh`. You could find out the name of new created environment with ``` From d817dfd84619aba5bf01309aacbff2f502093e00 Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Thu, 25 Jan 2024 13:19:37 -0500 Subject: [PATCH 28/29] Add .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7925ec0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +tmpfs +log_* +toast_test_output From d472297b5a6af86d01a25066c8fcdf0a44fa9f03 Mon Sep 17 00:00:00 2001 From: Bai-Chiang Date: Fri, 26 Jan 2024 09:07:55 -0500 Subject: [PATCH 29/29] Update install.md unify code block style --- docs/docs/install.md | 102 ++++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 49 deletions(-) diff --git a/docs/docs/install.md b/docs/docs/install.md index 4c7efa9..1f097f0 100644 --- a/docs/docs/install.md +++ b/docs/docs/install.md @@ -5,7 +5,7 @@ The install script takes options to specify the location of the base environment to create. It also allows specifying a central location to install the modulefile: - $> ./soconda.sh -h + ./soconda.sh -h Usage: ./soconda.sh [-c ] [-e ] @@ -33,9 +33,9 @@ This will greatly speed up the dependency resolution calculation. For new installation run following script to install miniforge -``` -./tools/bootstrap_base "$HOME/miniforge3" -``` + + ./tools/bootstrap_base "$HOME/miniforge3" + This will intall conda to `$HOME/miniforge3` directory. It will set conda-forge as default channel and use `libmamba` as default solver. After the installation you need to re-login or start a new terminal to initialize conda. @@ -51,70 +51,69 @@ customized MPI compiler, then set the `MPICC` environment variable to the MPI C compiler before running `soconda.sh`. That will cause the mpi4py package to be built using your system MPI compiler. -## Example: Local System +## Install soconda +### Example: Local System This installation could install `soconda` to your local computer and any cluster. Clone soconda repo -``` -git clone git@github.com:simonsobs/soconda.git -cd soconda -``` + + git clone git@github.com:simonsobs/soconda.git + cd soconda Run the `soconda.sh` script -``` -export MAKEFLAGS='-j 4' -bash soconda.sh -e soconda -c default -``` + + export MAKEFLAGS='-j 4' + bash soconda.sh -e soconda -c default + This will create a new environment `soconda_xxx.x.x` with version number as suffix using `default` configuration. [More details on configuration.](#customizing-an-environment) (The `MAKEFLAGS` doesn't seem to have any effect.) If you want to specify a conda base directory add `-b "$HOME/miniforge3"` argument to `soconda.sh`. You could find out the name of new created environment with -``` -conda env list -``` + + conda env list Then you can now activate the environment with -``` -conda activate soconda_xxx.x.x -``` + + conda activate soconda_xxx.x.x + If running on a Linux desktop that uses wayland, you also need to install the `qt-wayland` package -``` -conda install qt-wayland -``` + + conda install qt-wayland + If running on server, start jupyterlab listening on port `12345` with command -``` -cd /path/to/project -nohup jupyter-lab --no-browser --port=12345 &> jupyter.log & -``` + + cd /path/to/project + nohup jupyter-lab --no-browser --port=12345 &> jupyter.log & + To list current running jupyter server: -``` -jupyter server list -``` + + jupyter server list + To connect to jupyterlab running on server, start SSH tunnel from your laptop/desktop: -``` -ssh -N -L 12345:localhost:12345 server_domain_or_ip -``` + + ssh -N -L 12345:localhost:12345 server_domain_or_ip + Then you can connect to jupyterlab with link provided by command `jupyter server list`. To stop jupyterlab listenging on port 12345: -``` -jupyter server stop 12345 -``` -## Example: NERSC + jupyter server stop 12345 + + +### Example: NERSC At NERSC, the default provided python is from Anaconda, and does not work well for our needs. Instead, we have a conda-forge base system installed in our project software directory: - $> source /global/common/software/sobs/perlmutter/conda/etc/profile.d/conda.sh - $> conda activate base + source /global/common/software/sobs/perlmutter/conda/etc/profile.d/conda.sh + conda activate base Now we can either install a shared software environment or use this base environment to build a conda environment in a personal directory. If you are @@ -123,29 +122,29 @@ account and follow a specific naming convention which is beyond the scope of this document. If you wanted to install these tools to your home directory you could do: - $> mkdir -p ~/conda_envs - $> ./soconda.sh -e ~/conda_envs/soconda -m ~/conda_envs/modulefiles + mkdir -p ~/conda_envs + ./soconda.sh -e ~/conda_envs/soconda -m ~/conda_envs/modulefiles And then load the module: - $> module use ~/conda_envs/modulefiles - $> module avail - $> module load soconda/XXXXXX + module use ~/conda_envs/modulefiles + module avail + module load soconda/XXXXXX ## Running Tests After loading an `soconda` environment, you can run some tests with: - $> ./run_tests.sh + ./run_tests.sh -## Installing a Jupyter Kernel +## Installing a Jupyter Kernel for external Jupyter server -After loading an soconda module the first time, you can run (once) the included script: +After loading an soconda module or activate socond conda environment the first time, you can run (once) the included script: - $> soconda_jupyter.sh + soconda_jupyter.sh -This will install a kernel file so that jupyter knows how to launch a kernel -using this python stack. +This will install a kernel file to `~/.local/share/jupyter/kernels/soconda-xxxxx` so that external jupyter server +knows how to launch a kernel using this python stack. ## Customizing an Environment @@ -165,6 +164,11 @@ running command `conda remove --name envname --all` or `conda remove -p /base_di Or directly removing the `/envs/` directory. You can optionally delete the modulefile and the versioned pip local directory in your home directory. +If you installed a jupyter kernel, remove kernel file in `~/.local/share/jupyter/kernels/` +with matching soconda version. +If you have multiple soconda environment and deleted wrong kernel file, you can always +[recreate it](#installing-a-jupyter-kernel). + ## Advanced Details