diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7925ec0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +tmpfs +log_* +toast_test_output diff --git a/config/binary/packages_conda.txt b/config/binary/packages_conda.txt new file mode 100644 index 0000000..48fc35e --- /dev/null +++ b/config/binary/packages_conda.txt @@ -0,0 +1,7 @@ +# Adjust the default python version +python=3.10.* +# +jupyterlab +wurlitzer +plotly +plotly-resampler diff --git a/config/binary/packages_local.txt b/config/binary/packages_local.txt new file mode 100644 index 0000000..e69de29 diff --git a/config/binary/packages_pip.txt b/config/binary/packages_pip.txt new file mode 100644 index 0000000..5653739 --- /dev/null +++ b/config/binary/packages_pip.txt @@ -0,0 +1,17 @@ +pixell +quaternionarray +pysm3 +# Uncomment here if installing wheel, and comment out +# line in packages_local.txt +posix-ipc +toast==3.0.0a20 +pysqlite3-wheels +qpoint +so3g +# +# Latest tag +https://github.com/simonsobs/sotodlib/archive/master.tar.gz +# +# The mapsims package has hard-coded versions of pysm3 and +# pixell, which un-installs our versions of those packages. +#https://github.com/galsci/mapsims/archive/main.tar.gz diff --git a/config/binary/post_install.sh b/config/binary/post_install.sh new file mode 100644 index 0000000..5b3bcce --- /dev/null +++ b/config/binary/post_install.sh @@ -0,0 +1,15 @@ +# This script is sourced in the main `soconda.sh` script. +# Any commands placed here will be executed within that shell +# and have access to all environment variables defined there. +# There are 2 variables defined here which control the optional +# creation of a modulefile to load the environment and also +# which create a small script that installs a jupyter kernel +# for the environment into a user's home directory. + +# Install a module file? +install_module=yes + +# Install jupyter kernel setup script? +install_jupyter_setup=yes + +# Add any other shell commands here for this system... diff --git a/config/common.txt b/config/common.txt index 9d80cc2..f601f9f 100644 --- a/config/common.txt +++ b/config/common.txt @@ -5,6 +5,7 @@ # and parse pip dependencies with pipgrip. # conda-build +conda-verify compilers setuptools pip diff --git a/config/minimal/packages_conda.txt b/config/minimal/packages_conda.txt index 27fe859..0595a81 100644 --- a/config/minimal/packages_conda.txt +++ b/config/minimal/packages_conda.txt @@ -1,3 +1,3 @@ # Adjust the default python version -python=3.10.* +python=3.11.* # diff --git a/config/minimal/packages_local.txt b/config/minimal/packages_local.txt index 364899e..e69de29 100644 --- a/config/minimal/packages_local.txt +++ b/config/minimal/packages_local.txt @@ -1,13 +0,0 @@ -# This config skips all the ACT tools and just installs -# binary packages for everything. -#qpoint -#pixell -#libactpol_deps -#libactpol -#moby2 -# Remove next line once toast-3.0 is on conda-forge, -# and move to packages_conda.txt -#toast -# Uncomment next line to build so3g from source, and -# comment out line in packages_pip.txt. -#so3g diff --git a/config/minimal/packages_pip.txt b/config/minimal/packages_pip.txt index 768920a..5653739 100644 --- a/config/minimal/packages_pip.txt +++ b/config/minimal/packages_pip.txt @@ -3,12 +3,14 @@ quaternionarray pysm3 # Uncomment here if installing wheel, and comment out # line in packages_local.txt -pysqlite3-wheels +posix-ipc toast==3.0.0a20 +pysqlite3-wheels +qpoint so3g # # Latest tag -https://github.com/simonsobs/sotodlib/archive/refs/master.tar.gz +https://github.com/simonsobs/sotodlib/archive/master.tar.gz # # The mapsims package has hard-coded versions of pysm3 and # pixell, which un-installs our versions of those packages. diff --git a/config/minimal/post_install.sh b/config/minimal/post_install.sh index f640d65..5b3bcce 100644 --- a/config/minimal/post_install.sh +++ b/config/minimal/post_install.sh @@ -7,9 +7,9 @@ # for the environment into a user's home directory. # Install a module file? -install_module=no +install_module=yes # Install jupyter kernel setup script? -install_jupyter_setup=no +install_jupyter_setup=yes # Add any other shell commands here for this system... diff --git a/config/perlmutter/packages_pip.txt b/config/perlmutter/packages_pip.txt index 066af03..ae03dcd 100644 --- a/config/perlmutter/packages_pip.txt +++ b/config/perlmutter/packages_pip.txt @@ -7,7 +7,7 @@ pysm3 #so3g # # Latest tag -https://github.com/simonsobs/sotodlib/archive/refs/master.tar.gz +https://github.com/simonsobs/sotodlib/archive/master.tar.gz # # The mapsims package has hard-coded versions of pysm3 and # pixell, which un-installs our versions of those packages. diff --git a/config/site/module_init b/config/site/module_init new file mode 100644 index 0000000..c4cd8c8 --- /dev/null +++ b/config/site/module_init @@ -0,0 +1 @@ +-- No module dependencies diff --git a/config/site/packages_conda.txt b/config/site/packages_conda.txt new file mode 100644 index 0000000..45632ed --- /dev/null +++ b/config/site/packages_conda.txt @@ -0,0 +1,6 @@ +# Adjust the default python version +python=3.10.* +# +wurlitzer +plotly +plotly-resampler diff --git a/config/site/packages_local.txt b/config/site/packages_local.txt new file mode 100644 index 0000000..352fd00 --- /dev/null +++ b/config/site/packages_local.txt @@ -0,0 +1,11 @@ +qpoint +pixell +libactpol_deps +libactpol +moby2 +# Remove next line once toast-3.0 is on conda-forge, +# and move to packages_conda.txt +toast +# Uncomment next line to build so3g from source, and +# comment out line in packages_pip.txt. +so3g diff --git a/config/site/packages_pip.txt b/config/site/packages_pip.txt new file mode 100644 index 0000000..ae03dcd --- /dev/null +++ b/config/site/packages_pip.txt @@ -0,0 +1,14 @@ +quaternionarray +pysm3 +# +# Uncomment here if installing wheel, and comment out +# line in packages_local.txt +#pysqlite3-wheels +#so3g +# +# Latest tag +https://github.com/simonsobs/sotodlib/archive/master.tar.gz +# +# The mapsims package has hard-coded versions of pysm3 and +# pixell, which un-installs our versions of those packages. +#https://github.com/galsci/mapsims/archive/main.tar.gz diff --git a/config/site/post_install.sh b/config/site/post_install.sh new file mode 100644 index 0000000..5b3bcce --- /dev/null +++ b/config/site/post_install.sh @@ -0,0 +1,15 @@ +# This script is sourced in the main `soconda.sh` script. +# Any commands placed here will be executed within that shell +# and have access to all environment variables defined there. +# There are 2 variables defined here which control the optional +# creation of a modulefile to load the environment and also +# which create a small script that installs a jupyter kernel +# for the environment into a user's home directory. + +# Install a module file? +install_module=yes + +# Install jupyter kernel setup script? +install_jupyter_setup=yes + +# Add any other shell commands here for this system... diff --git a/docs/docs/install.md b/docs/docs/install.md index 47ce0fd..1f097f0 100644 --- a/docs/docs/install.md +++ b/docs/docs/install.md @@ -5,7 +5,7 @@ The install script takes options to specify the location of the base environment to create. It also allows specifying a central location to install the modulefile: - $> ./soconda.sh -h + ./soconda.sh -h Usage: ./soconda.sh [-c ] [-e ] @@ -25,22 +25,20 @@ clutter. ## Base Conda Environment -If you already have a conda-forge base environment, then you can skip this +If you already have a conda-forge or micromamba base environment, then you can skip this step. However, you should consider setting the "solver" in the base environment -to use libmamba. This will greatly speed up the dependency resolution -calculation. Once you decide on the install prefix for your overall conda -environment you can use the included bootstrap script. For this example, we -will use `/opt/conda` as the path to the conda base installation. Now run the -bootstrap script: +to use `libmamba`. To use `libmamba` solver see +[this](https://www.anaconda.com/blog/a-faster-conda-for-a-growing-community) article. +This will greatly speed up the dependency resolution +calculation. - $> ./tools/bootstrap_base "/opt/conda" +For new installation run following script to install miniforge -This bootstrap will install a base system with the conda-forge channel set to -the default and using the libmamba solver. You can now source the conda -initialization file and activate this base environment: + ./tools/bootstrap_base "$HOME/miniforge3" - $> source /opt/conda/etc/profile.d/conda.sh - $> conda activate base +This will intall conda to `$HOME/miniforge3` directory. +It will set conda-forge as default channel and use `libmamba` as default solver. +After the installation you need to re-login or start a new terminal to initialize conda. After installing an `soconda` environment below, you will not need this step since it is done by the generated modulefile. @@ -53,34 +51,69 @@ customized MPI compiler, then set the `MPICC` environment variable to the MPI C compiler before running `soconda.sh`. That will cause the mpi4py package to be built using your system MPI compiler. -## Example: Local System +## Install soconda +### Example: Local System +This installation could install `soconda` to your local computer and any cluster. -Starting from scratch, bootstrap a small conda-forge base environment in `~/conda`: +Clone soconda repo - $> ./tools/bootstrap_base.sh ~/conda - $> source ~/conda/etc/profile.d/conda.sh - $> conda activate base + git clone git@github.com:simonsobs/soconda.git + cd soconda -Create an `soconda` environment with default name and version. However, we -decide to put all the modulefiles into a central location in the root of the -base conda install: +Run the `soconda.sh` script - $> ./soconda.sh -b ~/conda -m ~/conda/modulefiles + export MAKEFLAGS='-j 4' + bash soconda.sh -e soconda -c default -Now we can load the module: +This will create a new environment `soconda_xxx.x.x` with version number as suffix +using `default` configuration. [More details on configuration.](#customizing-an-environment) +(The `MAKEFLAGS` doesn't seem to have any effect.) +If you want to specify a conda base directory add `-b "$HOME/miniforge3"` argument to `soconda.sh`. - $> module use ~/conda/modulefiles - $> module avail - $> module load soconda/XXXXXX +You could find out the name of new created environment with -## Example: NERSC + conda env list + +Then you can now activate the environment with + + conda activate soconda_xxx.x.x + + +If running on a Linux desktop that uses wayland, you also need to install the `qt-wayland` package + + conda install qt-wayland + + +If running on server, start jupyterlab listening on port `12345` with command + + cd /path/to/project + nohup jupyter-lab --no-browser --port=12345 &> jupyter.log & + + +To list current running jupyter server: + + jupyter server list + + +To connect to jupyterlab running on server, start SSH tunnel from your laptop/desktop: + + ssh -N -L 12345:localhost:12345 server_domain_or_ip + +Then you can connect to jupyterlab with link provided by command `jupyter server list`. + +To stop jupyterlab listenging on port 12345: + + jupyter server stop 12345 + + +### Example: NERSC At NERSC, the default provided python is from Anaconda, and does not work well for our needs. Instead, we have a conda-forge base system installed in our project software directory: - $> source /global/common/software/sobs/perlmutter/conda/etc/profile.d/conda.sh - $> conda activate base + source /global/common/software/sobs/perlmutter/conda/etc/profile.d/conda.sh + conda activate base Now we can either install a shared software environment or use this base environment to build a conda environment in a personal directory. If you are @@ -89,29 +122,29 @@ account and follow a specific naming convention which is beyond the scope of this document. If you wanted to install these tools to your home directory you could do: - $> mkdir -p ~/conda_envs - $> ./soconda.sh -e ~/conda_envs/soconda -m ~/conda_envs/modulefiles + mkdir -p ~/conda_envs + ./soconda.sh -e ~/conda_envs/soconda -m ~/conda_envs/modulefiles And then load the module: - $> module use ~/conda_envs/modulefiles - $> module avail - $> module load soconda/XXXXXX + module use ~/conda_envs/modulefiles + module avail + module load soconda/XXXXXX ## Running Tests After loading an `soconda` environment, you can run some tests with: - $> ./run_tests.sh + ./run_tests.sh -## Installing a Jupyter Kernel +## Installing a Jupyter Kernel for external Jupyter server -After loading an soconda module the first time, you can run (once) the included script: +After loading an soconda module or activate socond conda environment the first time, you can run (once) the included script: - $> soconda_jupyter.sh + soconda_jupyter.sh -This will install a kernel file so that jupyter knows how to launch a kernel -using this python stack. +This will install a kernel file to `~/.local/share/jupyter/kernels/soconda-xxxxx` so that external jupyter server +knows how to launch a kernel using this python stack. ## Customizing an Environment @@ -127,9 +160,15 @@ Then install it as usual. ## Deleting an Environment The `soconda` environments are self contained and you can delete them by -removing the path or (if using a name), removing the `/envs/` directory. You can optionally delete the modulefile and the versioned pip +running command `conda remove --name envname --all` or `conda remove -p /base_dir/envs/name --all`. +Or directly removing the `/envs/` directory. +You can optionally delete the modulefile and the versioned pip local directory in your home directory. +If you installed a jupyter kernel, remove kernel file in `~/.local/share/jupyter/kernels/` +with matching soconda version. +If you have multiple soconda environment and deleted wrong kernel file, you can always +[recreate it](#installing-a-jupyter-kernel). + ## Advanced Details diff --git a/pkgs/so3g/meta.yaml b/pkgs/so3g/meta.yaml index 15aa6f2..796d128 100644 --- a/pkgs/so3g/meta.yaml +++ b/pkgs/so3g/meta.yaml @@ -1,5 +1,5 @@ {% set version = "0.1.10" %} -{% set sha256 = "4cabc243acf6a18bb8b63572f468b303c3bc4cb4e0e901e3fbe222a8ff094e11" %} +{% set sha256 = "5367cdd24ac7eff70b482ede2c32df8e1acd16e582a24e489467ec2c4760c31b" %} {% set build = 0 %} diff --git a/pkgs/toast/meta.yaml b/pkgs/toast/meta.yaml index 2fc5c1a..39a903b 100644 --- a/pkgs/toast/meta.yaml +++ b/pkgs/toast/meta.yaml @@ -1,6 +1,6 @@ -{% set version = "3980cbb579de56a379df000df76e69a1c1ac73ae" %} -{% set sha256 = "e2713145f758631cd6bb24b89a87f2522322a72a29400f8ba51d5144e9afe54d" %} +{% set version = "3.0.0a20" %} +{% set sha256 = "4d5ad584bf0d2e851009beb17a4ec96677b36b9c9e18893910014581bc22bb4c" %} {% set build = 0 %} @@ -9,7 +9,7 @@ package: version: {{ version }} source: - url: https://github.com/hpc4cmb/toast/archive/{{ version }}.tar.gz + url: https://github.com/hpc4cmb/toast/archive/refs/tags/{{ version }}.tar.gz sha256: {{ sha256 }} # patches: # - debug_lapack.patch diff --git a/run_tests.sh b/run_tests.sh index 3dd7c71..55ed4b4 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,10 +1,10 @@ #!/bin/bash # Pixell Tests -OMP_NUM_THREADS=4 test_pixell.py +#OMP_NUM_THREADS=4 test_pixell.py # Serial TOAST Tests OMP_NUM_THREADS=4 MPI_DISABLED=1 python -c 'import toast.tests; toast.tests.run()' # MPI-enabled tests -OMP_NUM_THREADS=2 mpirun -np 4 python -c 'import toast.tests; toast.tests.run()' +OMP_NUM_THREADS=2 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()' diff --git a/soconda.sh b/soconda.sh index 4dcbbdd..b9d4018 100755 --- a/soconda.sh +++ b/soconda.sh @@ -69,6 +69,23 @@ if [ -z "${envname}" ]; then echo "Environment root name not specified, using \"soconda\"" envname="soconda" fi +# The full environment name, including the root and version. +fullenv="${envname}_${version}" +# Determine whether the new environment is a name or a full path. +env_noslash=$(echo "${fullenv}" | sed -e 's/\///g') +if [ "${env_noslash}" != "${fullenv}" ]; then + # This was a path + is_path='yes' + # Make sure this is a full path + if [[ "${envname:0:1}" != '/' ]] ; then + # The path does not start at root + echo "Please provide a full path." + exit 1 + fi +else + is_path='no' +fi + if [ -z "${config}" ]; then echo "No config specified, using \"default\"" @@ -77,12 +94,6 @@ else echo "Using config \"${config}\"" fi -# The env root name, used for the name of the generated module file -envroot=$(basename ${envname}) - -# The full environment name, including the root and version. -fullenv="${envname}_${version}" - # The path to the selected config directory confdir="${scriptdir}/config/${config}" if [ ! -d "${confdir}" ]; then @@ -90,36 +101,49 @@ if [ ! -d "${confdir}" ]; then exit 1 fi -# The optional module init for this config -modinit="${confdir}/module_init" -# Activate the base environment if [ -n "${base}" ]; then conda_dir="${base}" + # Initialize conda + source "${conda_dir}/etc/profile.d/conda.sh" + # Conda initialization script will create conda function, + # but micromamba will create micromamba function. + # Here create a conda_exec function that unify conda and micromamba + # installation. + conda_exec () { conda "$@" ; } else # User did not specify where to find it - if [ -z "$(which conda)" ]; then - # conda is not in the path + if [ -n "$CONDA_EXE" ]; then + echo "Find conda command at ${CONDA_EXE}" + conda_dir="$(dirname $(dirname $CONDA_EXE))" + # Initialize conda + eval "$("$CONDA_EXE" 'shell.bash' 'hook')" + conda_exec () { conda "$@" ; } + elif [ -n "$MAMBA_EXE" ]; then + echo "Find micromamba command at ${MAMBA_EXE}" + conda_dir="$MAMBA_ROOT_PREFIX" + # Initialize micromamba + eval "$("$MAMBA_EXE" shell hook --shell bash)" + conda_exec () { micromamba "$@" ; } + else + # Could not find conda or micromamba echo "You must either activate the conda base environment before" echo "running this script, or you must specify the path to the base" echo "install with the \"-b \" option." exit 1 - else - # Conda is in the path - conda_dir="$(dirname $(dirname $(which conda)))" fi fi -# Make sure that conda is initialized -source "${conda_dir}/etc/profile.d/conda.sh" -conda activate base # Conda package cache. In some cases this can get really big, so # we point it to a temp location unless the user has overridden it. if [ -z ${CONDA_PKGS_DIRS} ]; then if [ -z ${NERSC_HOST} ]; then # Standard system - conda_tmp=$(mktemp -d) + # Create temp direcotry + mkdir -p "$scriptdir/tmpfs" + conda_tmp=$(mktemp -d --tmpdir="$scriptdir/tmpfs") + export TMPDIR="$conda_tmp" else # Running at NERSC, use a directory in scratch conda_tmp="${SCRATCH}/tmp_soconda" @@ -128,36 +152,25 @@ if [ -z ${CONDA_PKGS_DIRS} ]; then export CONDA_PKGS_DIRS="${conda_tmp}" fi -# Determine whether the new environment is a name or a full path. -env_noslash=$(echo "${fullenv}" | sed -e 's/\///g') -is_path=no -if [ "${env_noslash}" != "${fullenv}" ]; then - # This was a path - is_path=yes - env_check="" - if [ -e "${fullenv}/bin/conda" ]; then - # It already exists - env_check="${fullenv}" - fi -else - env_check=$(conda env list | grep "${fullenv} ") -fi # Extract the python version we are using (if not the default) # and pass that to the conda create command. python_version=$(cat "${confdir}/packages_conda.txt" | grep 'python=') +# Check this env exist or not +# env_check would be empty if not exist +env_check=$(conda_exec env list | grep "${fullenv}") +echo -e "\n\n" if [ -z "${env_check}" ]; then - # Environment does not yet exist. Create it. if [ ${is_path} = "no" ]; then echo "Creating new environment \"${fullenv}\"" - conda create --yes -n "${fullenv}" "${python_version}" + conda_exec create --yes -n "${fullenv}" "${python_version}" else echo "Creating new environment \"${fullenv}\"" - conda create --yes -p "${fullenv}" "${python_version}" + conda_exec create --yes -p "${fullenv}" "${python_version}" fi echo "Activating environment \"${fullenv}\"" - conda activate "${fullenv}" + conda_exec activate "${fullenv}" # Create condarc for this environment echo "# condarc for soconda" > "${CONDA_PREFIX}/.condarc" @@ -169,70 +182,61 @@ if [ -z "${env_check}" ]; then echo "solver: libmamba" >> "${CONDA_PREFIX}/.condarc" # Reactivate to pick up changes - conda deactivate - conda activate "${fullenv}" + conda_exec deactivate + conda_exec activate "${fullenv}" # Copy logo files cp "${scriptdir}"/logo* "${CONDA_PREFIX}/" else echo "Activating environment \"${fullenv}\"" - conda activate "${fullenv}" - conda env list + conda_exec activate "${fullenv}" fi +conda_exec env list -# Install conda packages. - -conda_pkgs="" -for pkgfile in "${scriptdir}/config/common.txt" "${confdir}/packages_conda.txt"; do - while IFS='' read -r line || [[ -n "${line}" ]]; do - # Is this line commented? - comment=$(echo "${line}" | cut -c 1) - if [ "${comment}" != "#" ]; then - pkgname="${line}" - conda_pkgs="${conda_pkgs} ${pkgname}" - fi - done < "${pkgfile}" -done +# Install conda packages. +echo -e "\n\n" echo "Installing conda packages..." | tee "log_conda" -conda install --yes ${conda_pkgs} \ - | tee -a "log_conda" 2>&1 +conda_exec install --yes --file "${scriptdir}/config/common.txt" \ + |& tee -a "log_conda" +conda_exec install --yes --file "${confdir}/packages_conda.txt" \ + |& tee -a "log_conda" # The "cc" symlink from the compilers package shadows Cray's MPI C compiler... rm -f "${CONDA_PREFIX}/bin/cc" -conda deactivate -conda activate "${fullenv}" +conda_exec deactivate +conda_exec activate "${fullenv}" -# Use pipgrip to install dependencies of pip packages with conda. -pip install pipgrip - -mkdir -p "${CONDA_PREFIX}/conda-bld" -conda-index "${CONDA_PREFIX}/conda-bld" -conda config \ - --file "${CONDA_PREFIX}/.condarc" \ - --add channels "file://${CONDA_PREFIX}/conda-bld" - -# Get the python site packages version -pyver=$(python3 --version 2>&1 | awk '{print $2}' | sed -e "s#\(.*\)\.\(.*\)\..*#\1.\2#") # Install mpi4py - +echo -e "\n\n" echo "Installing mpi4py..." | tee "log_mpi4py" if [ -z "${MPICC}" ]; then echo "The MPICC environment variable is not set. Installing mpi4py" \ | tee -a "log_mpi4py" echo "from the conda package, rather than building from source." \ | tee -a "log_mpi4py" - conda install --yes openmpi mpi4py | tee -a "log_mpi4py" 2>&1 \ + conda_exec install --yes openmpi mpi4py |& tee -a "log_mpi4py" \ # Disable the ancient openib btl, in order to avoid a harmless warning echo 'btl = ^openib' >> "${CONDA_PREFIX}/etc/openmpi-mca-params.conf" else echo "Building mpi4py with MPICC=\"${MPICC}\"" | tee -a "log_mpi4py" pip install --force-reinstall --no-cache-dir --no-binary=mpi4py mpi4py \ - | tee -a "log_mpi4py" 2>&1 + |& tee -a "log_mpi4py" fi + # Build local packages +# Here we use conda instead of conda_exec, because conda is a dependency +# of conda-build package. Therefore after activated ${fullenv}, +# conda command is availabe in both micromamba and miniforge installation. +# If you run $(which conda) command it should return $CONDA_PREFIX/bin/conda +# in both cases. +mkdir -p "${CONDA_PREFIX}/conda-bld" +conda index "${CONDA_PREFIX}/conda-bld" +conda config \ + --file "${CONDA_PREFIX}/.condarc" \ + --add channels "file://${CONDA_PREFIX}/conda-bld" while IFS='' read -r line || [[ -n "${line}" ]]; do # Is this line commented? @@ -240,57 +244,75 @@ while IFS='' read -r line || [[ -n "${line}" ]]; do if [ "${comment}" != "#" ]; then pkgname="${line}" pkgrecipe="${scriptdir}/pkgs/${pkgname}" - echo "Building local package '${pkgname}'" | tee "log_${pkgname}" 2>&1 - conda-build ${pkgrecipe} | tee -a "log_${pkgname}" 2>&1 - echo "Installing local package '${pkgname}'" | tee -a "log_${pkgname}" 2>&1 - conda install --yes --use-local ${pkgname} | tee -a "log_${pkgname}" 2>&1 + echo -e "\n\n" + echo "Building local package '${pkgname}'" |& tee "log_${pkgname}" + conda build ${pkgrecipe} |& tee -a "log_${pkgname}" + echo "Installing local package '${pkgname}'" |& tee -a "log_${pkgname}" + conda install --yes --use-local ${pkgname} |& tee -a "log_${pkgname}" fi done < "${confdir}/packages_local.txt" +echo -e "\n\n" echo "Cleaning up build products" -conda-build purge +conda build purge + +# Remove buid directory +rm -rf "${conda_tmp}" &> /dev/null + +# Remove /tmp/pixell-* test files create by pixell/setup.py +find /tmp -maxdepth 1 -type f -name 'pixell-*' -exec rm {} \; + # Install pip packages. We install one package at a time # with no dependencies, so that we will intentionally # get an error. All dependency packages should be installed # through conda. +# Use pipgrip to install dependencies of pip packages with conda. +echo -e "\n\n" +pip install pipgrip + +echo -e "\n" echo "Installing pip packages..." | tee "log_pip" +installed_pkgs="$(conda_exec list | awk '{print $1}')" while IFS='' read -r line || [[ -n "${line}" ]]; do - # Is this line commented? - comment=$(echo "${line}" | cut -c 1) - if [ "${comment}" != "#" ]; then + # Skip if $line is empty + # If the $line start with '#' then it's a comment. + if [[ -n "${line}" && "${line:0:1}" != "#" ]]; then pkg="${line}" url_check=$(echo "${pkg}" | grep '/') - if [ "x${url_check}" = "x" ]; then - echo "Checking dependencies for package \"${pkg}\"" | tee -a "log_pip" 2>&1 + if [ -z "${url_check}" ]; then + echo "Checking dependencies for package \"${pkg}\"" |& tee -a "log_pip" pkgbase=$(echo ${pkg} | sed -e 's/\([[:alnum:]_\-]*\).*/\1/') - for dep in $(pipgrip --pipe "${pkg}"); do + for dep in $(pipgrip --pipe --threads 4 "${pkg}"); do name=$(echo ${dep} | sed -e 's/\([[:alnum:]_\-]*\).*/\1/') if [ "${name}" != "${pkgbase}" ]; then - depcheck=$(conda list ${name} | awk '{print $1}' | grep -E "^${name}\$") + depcheck=$(echo "$installed_pkgs" | grep -E "^${name}\$") if [ -z "${depcheck}" ]; then # It is not already installed, try to install it with conda - echo "Attempt to install conda package for dependency \"${name}\"..." | tee -a "log_pip" 2>&1 - conda install --yes ${name} | tee -a "log_pip" 2>&1 - if [ $? -ne 0 ]; then - echo " No conda package available for dependency \"${name}\"" | tee -a "log_pip" 2>&1 - echo " Assuming pip package already installed." | tee -a "log_pip" 2>&1 - fi + echo "Attempt to install conda package for dependency \"${name}\"..." |& tee -a "log_pip" + conda_exec install --yes ${name} |& tee -a "log_pip" + installed_pkgs="${installed_pkgs}"$'\n'"${name}" else - echo " Package for dependency \"${name}\" already installed" | tee -a "log_pip" 2>&1 + echo " Package for dependency \"${name}\" already installed" |& tee -a "log_pip" fi fi done else - echo "Pip package \"${pkg}\" is a URL, skipping dependency check" | tee -a "log_pip" 2>&1 + echo "Pip package \"${pkg}\" is a URL, skipping dependency check" |& tee -a "log_pip" fi - echo "Installing package ${pkg} with --no-deps" | tee -a "log_pip" 2>&1 - python3 -m pip install --no-deps ${pkg} | tee -a "log_pip" 2>&1 + echo "Installing package ${pkg} with --no-deps" |& tee -a "log_pip" + python3 -m pip install --no-deps ${pkg} |& tee -a "log_pip" + installed_pkgs="${installed_pkgs}"$'\n'"${pkg}" + echo -e "\n\n" fi done < "${confdir}/packages_pip.txt" + +# Get the python site packages version +pyver=$(python3 --version 2>&1 | awk '{print $2}' | sed -e "s#\(.*\)\.\(.*\)\..*#\1.\2#") + # Subsitutions to use when parsing input templates confsub="-e 's#@VERSION@#${version}#g'" confsub="${confsub} -e 's#@BASE@#${conda_dir}#g'" diff --git a/tools/install_modulefile.sh b/tools/install_modulefile.sh index 840e289..bb20f8f 100644 --- a/tools/install_modulefile.sh +++ b/tools/install_modulefile.sh @@ -1,6 +1,12 @@ # Modulefile install snippet. This is sourced from the main # `soconda.sh` script and uses variables defined there. +# The env root name, used for the name of the generated module file +envroot=$(basename ${envname}) + +# The optional module init for this config +modinit="${confdir}/module_init" + if [ -z "${moduledir}" ]; then # No centralized directory was specified for modulefiles. Make # a subdirectory within the environment itself.