Merge pull request #740 from jashapiro/711-713-docker-gha-miniforge

Update docker images and GHA to use miniforge
AlexsLemonade · Aug 30, 2024 · 436026b · 436026b
2 parents 402371e + b87bdae
commit 436026b
Show file tree

Hide file tree

Showing 8 changed files with 61 additions and 44 deletions.
diff --git a/.github/components/dictionary.txt b/.github/components/dictionary.txt
@@ -115,6 +115,7 @@ mesenchymal
 metacell
 metacells
 Miniconda
+Miniforge
 misidentification
 MSC
 multifactor
@@ -154,11 +155,11 @@ scDblFinder
 SCE
 ScPCA
 SCPCP
+scRNA
 scrublet
 SEACells
 SemVar
 SingleR
-scRNA
 snRNA
 socio
 SSO

diff --git a/.github/workflows/run_hello-python.yml b/.github/workflows/run_hello-python.yml
@@ -37,6 +37,8 @@ jobs:
 
       - name: Set up conda
         uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
 
       - name: Install and activate locked conda environment
         run: |

diff --git a/.github/workflows/run_metacells.yml b/.github/workflows/run_metacells.yml
@@ -44,6 +44,8 @@ jobs:
 
       - name: Set up conda
         uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
 
       - name: Install and activate locked conda environment
         run: |

diff --git a/analyses/cell-type-ewings/Dockerfile b/analyses/cell-type-ewings/Dockerfile
@@ -16,7 +16,7 @@ ENV RENV_CONFIG_CACHE_ENABLED=FALSE
 # set a name for the conda environment
 ARG ENV_NAME=openscpca-cell-type-ewings
 
-# set environment variables to install miniconda
+# set environment variables to install conda
 ENV PATH="/opt/conda/bin:${PATH}"
 
 # Install dependencies for renv library
@@ -27,18 +27,20 @@ RUN apt-get -y update &&  \
   wget \
   && rm -rf /var/lib/apt/lists/*
 
-# Install miniconda
-# adapted from https://github.com/ContinuumIO/docker-images/blob/main/miniconda3/debian/Dockerfile
-RUN curl -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
-  && bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda \
-  && rm -f Miniconda3-latest-Linux-x86_64.sh \
-  && ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh  \
-  && echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc  \
-  && find /opt/conda/ -follow -type f -name '*.a' -delete \
-  && find /opt/conda/ -follow -type f -name '*.js.map' -delete
+# Install conda via miniforge
+# adapted from https://github.com/conda-forge/miniforge-images/blob/master/ubuntu/Dockerfile
+RUN curl -L "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" -o /tmp/miniforge.sh \
+  && bash /tmp/miniforge.sh -b -p /opt/conda \
+  && rm -f /tmp/miniforge.sh \
+  && conda clean --tarballs --index-cache --packages --yes \
+  && find /opt/conda -follow -type f -name '*.a' -delete \
+  && find /opt/conda -follow -type f -name '*.pyc' -delete \
+  && conda clean --force-pkgs-dirs --all --yes \
+  && echo ". /opt/conda/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc \
+  && echo ". /opt/conda/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc
 
 # add conda-lock
-RUN conda install conda-lock
+RUN conda install conda-lock && conda clean --all --yes
 
 # Install renv to enable later package installation
 RUN Rscript -e "install.packages('renv')"
@@ -47,8 +49,8 @@ RUN Rscript -e "install.packages('renv')"
 COPY conda-lock.yml conda-lock.yml
 
 # restore from conda-lock.yml file and clean up to reduce image size
-RUN conda-lock install -n ${ENV_NAME} conda-lock.yml && \
-  conda clean --all --yes
+RUN conda-lock install -n ${ENV_NAME} conda-lock.yml \
+  && conda clean --all --yes
 
 # Activate conda environment on bash launch
 RUN echo "conda activate ${ENV_NAME}" >> ~/.bashrc

diff --git a/analyses/doublet-detection/Dockerfile b/analyses/doublet-detection/Dockerfile
@@ -11,21 +11,24 @@ ARG ENV_NAME=openscpca-doublet-detection
 # Set an environment variable to allow checking if we are in an OpenScPCA container
 ENV OPENSCPCA_DOCKER=TRUE
 
-# set environment variables to install miniconda
+# set environment variables to install conda
 ENV PATH="/opt/conda/bin:${PATH}"
 
-# Install Miniconda
-# adapted from https://github.com/ContinuumIO/docker-images/blob/main/miniconda3/debian/Dockerfile
-RUN curl -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
-  && bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda \
-  && ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh  \
-  && echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc  \
-  && rm -f Miniconda3-latest-Linux-x86_64.sh \
-  && find /opt/conda/ -follow -type f -name '*.a' -delete \
-  && find /opt/conda/ -follow -type f -name '*.js.map' -delete
+# Install conda via miniforge
+# adapted from https://github.com/conda-forge/miniforge-images/blob/master/ubuntu/Dockerfile
+RUN curl -L "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" -o /tmp/miniforge.sh \
+  && bash /tmp/miniforge.sh -b -p /opt/conda \
+  && rm -f /tmp/miniforge.sh \
+  && conda clean --tarballs --index-cache --packages --yes \
+  && find /opt/conda -follow -type f -name '*.a' -delete \
+  && find /opt/conda -follow -type f -name '*.pyc' -delete \
+  && conda clean --force-pkgs-dirs --all --yes \
+  && echo ". /opt/conda/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc \
+  && echo ". /opt/conda/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc
 
 # Install conda-lock
-RUN conda install --channel=conda-forge --name=base conda-lock
+RUN conda install --channel=conda-forge --name=base conda-lock \
+  && conda clean --all --yes
 
 # Install pandoc dependency
 RUN apt-get -y update &&  \
@@ -44,8 +47,11 @@ ENV RENV_CONFIG_CACHE_ENABLED=FALSE
 COPY conda-lock.yml conda-lock.yml
 
 # restore from conda-lock.yml file and clean up to reduce image size
-RUN conda-lock install -n ${ENV_NAME} conda-lock.yml && \
-  conda clean --all --yes
+RUN conda-lock install -n ${ENV_NAME} conda-lock.yml \
+  && conda clean --all --yes
+
+# Activate conda environment on bash launch
+RUN echo "conda activate ${ENV_NAME}" >> ~/.bashrc
 
 # Copy the renv.lock file from the host environment to the image
 COPY renv.lock renv.lock
@@ -65,8 +71,5 @@ RUN Rscript -e "proc <- basilisk::basiliskStart(env = zellkonverter::zellkonvert
   basilisk::basiliskStop(proc); \
   basilisk.utils::cleanConda()"
 
-# Activate conda environment on bash launch
-RUN echo "conda activate ${ENV_NAME}" >> ~/.bashrc
-
 # Set CMD to bash to activate the environment when launching
 CMD ["/bin/bash"]
diff --git a/docs/ensuring-repro/docker/docker-images.md b/docs/ensuring-repro/docker/docker-images.md
@@ -137,7 +137,7 @@ CMD ["/bin/bash"]
 If your analysis module requires both R and conda environments, you may have to do just a bit more work to set up the Dockerfile, as there is not a single base image that includes both R and conda environments.
 <!-- Should we make one, maybe? -->
 We recommend starting with an R-based image and then installing conda manually.
-In the example below, we have adapted the installation steps used in the [official Miniconda Dockerfile](https://github.com/ContinuumIO/docker-images/blob/main/miniconda3/debian/Dockerfile).
+In the example below, we have adapted the installation steps used in the [official Miniforge Dockerfile](https://github.com/conda-forge/miniforge-images/blob/master/ubuntu/Dockerfile).
 
 ```Dockerfile
 # Dockerfile for an analysis module with both R and conda environments
@@ -147,21 +147,24 @@ FROM bioconductor/r-ver:3.19
 # set a name for the conda environment
 ARG ENV_NAME=openscpca-analysis
 
-# set environment variables to install miniconda
+# set environment variables to install conda
 ENV PATH="/opt/conda/bin:${PATH}"
 
-# Install Miniconda
-# adapted from https://github.com/ContinuumIO/docker-images/blob/main/miniconda3/debian/Dockerfile
-RUN curl -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
-  && bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda \
-  && ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh  \
-  && echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc  \
-  && rm -f Miniconda3-latest-Linux-x86_64.sh \
-  && find /opt/conda/ -follow -type f -name '*.a' -delete \
-  && find /opt/conda/ -follow -type f -name '*.js.map' -delete
+# Install conda via miniforge
+# adapted from https://github.com/conda-forge/miniforge-images/blob/master/ubuntu/Dockerfile
+RUN curl -L "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" -o /tmp/miniforge.sh \
+  && bash /tmp/miniforge.sh -b -p /opt/conda \
+  && rm -f /tmp/miniforge.sh \
+  && conda clean --tarballs --index-cache --packages --yes \
+  && find /opt/conda -follow -type f -name '*.a' -delete \
+  && find /opt/conda -follow -type f -name '*.pyc' -delete \
+  && conda clean --force-pkgs-dirs --all --yes \
+  && echo ". /opt/conda/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc \
+  && echo ". /opt/conda/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc
 
 # Install conda-lock
-RUN conda install --channel=conda-forge --name=base conda-lock
+RUN conda install --channel=conda-forge --name=base conda-lock \
+  && conda clean --all --yes
 
 # Install renv
 RUN Rscript -e "install.packages('renv')"
@@ -173,8 +176,8 @@ ENV RENV_CONFIG_CACHE_ENABLED FALSE
 COPY conda-lock.yml conda-lock.yml
 
 # restore from conda-lock.yml file and clean up to reduce image size
-RUN conda-lock install -n ${ENV_NAME} conda-lock.yml && \
-  conda clean --all --yes
+RUN conda-lock install -n ${ENV_NAME} conda-lock.yml \
+  && conda clean --all --yes
 
 # Copy the renv.lock file from the host environment to the image
 COPY renv.lock renv.lock

diff --git a/templates/workflows/run_conda-module.yml b/templates/workflows/run_conda-module.yml
@@ -41,6 +41,8 @@ jobs:
       - name: Set up conda
         # Note that this creates and activates an environment named 'test' by default
         uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
 
       - name: Install and activate locked conda environment
         run: |

diff --git a/templates/workflows/run_conda-renv-module.yml b/templates/workflows/run_conda-renv-module.yml
@@ -55,6 +55,8 @@ jobs:
       - name: Set up conda
         # Note that this creates and activates an environment named 'test' by default
         uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
 
       - name: Install conda-lock and activate locked conda environment
         run: |