From 189ee8c31ed2e1bb97e7e4eb335e46239e533030 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 18 Mar 2024 16:21:30 +0100 Subject: [PATCH] ENH: Migrate to micromamba --- Dockerfile | 118 +++---------- env.yml | 31 ++++ requirements.txt | 443 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 499 insertions(+), 93 deletions(-) create mode 100644 env.yml create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index ff9e521..ab4fe42 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # Use Ubuntu 20.04 LTS -FROM ubuntu:focal-20210416 +FROM ubuntu:jammy-20240125 # Make apt non-interactive RUN echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90circleci \ @@ -35,6 +35,7 @@ RUN apt-get update && \ build-essential \ bzip2 \ ca-certificates \ + curl \ libtool \ locales \ lsb-release \ @@ -50,102 +51,33 @@ RUN locale-gen en_US.UTF-8 || true ENV LANG="en_US.UTF-8" \ LC_ALL="en_US.UTF-8" -# Leave these args here to better use the Docker build cache -# miniconda index: https://conda.io/en/latest/miniconda_hashes.html -ENV CONDA_PATH="/opt/conda" -ARG CONDA_VERSION=py39_23.1.0-1 -ARG SHA256SUM=5dc619babc1d19d6688617966251a38d245cb93d69066ccde9a013e1ebb5bf18 +WORKDIR / +RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh -O miniconda.sh && \ - echo "${SHA256SUM} miniconda.sh" > miniconda.sha256 && \ - sha256sum -c --status miniconda.sha256 && \ - mkdir -p /opt && \ - sh miniconda.sh -b -p ${CONDA_PATH} && \ - rm miniconda.sh miniconda.sha256 && \ - ln -s ${CONDA_PATH}/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". ${CONDA_PATH}/etc/profile.d/conda.sh" >> ~/.bashrc && \ - echo "conda activate base" >> ~/.bashrc && \ - find ${CONDA_PATH}/ -follow -type f -name '*.a' -delete && \ - find ${CONDA_PATH}/ -follow -type f -name '*.js.map' -delete && \ - ${CONDA_PATH}/bin/conda clean -afy - -# Set CPATH for packages relying on compiled libs (e.g. indexed_gzip) -ENV PATH="${CONDA_PATH}/bin:$PATH" \ - CPATH="${CONDA_PATH}/include:$CPATH" \ - PYTHONNOUSERSITE=1 - -COPY condarc /root/.condarc - -RUN ${CONDA_PATH}/bin/conda install mamba -n base && \ - mamba install -y \ - attrs=22.2 \ - codecov=2.1 \ - colorclass=2.2 \ - coverage=6.3 \ - curl=7.88 \ - datalad=0.18 \ - dipy=1.5 \ - flake8=4.0 \ - git=2.39 \ - graphviz=3.0 \ - h5py=3.8 \ - indexed_gzip=1.7 \ - jinja2=3.1 \ - libxml2=2.10 \ - libxslt=1.1 \ - lockfile=0.12 \ - matplotlib=3.5 \ - mkl=2022.1 \ - mkl-service=2.4 \ - nibabel=4.0 \ - nilearn=0.9 \ - nipype=1.8 \ - nitime=0.9 \ - nodejs=18 \ - numpy=1.24 \ - packaging=23.0 \ - pandas=1.5 \ - pandoc=2.19 \ - pbr=5.11 \ - pip=23.0 \ - pockets=0.9 \ - psutil=5.9 \ - pydot=1.4 \ - pytest=7.2 \ - pytest-cov=4.0 \ - pytest-env=0.8 \ - pytest-xdist=3.2 \ - pyyaml=6.0 \ - requests=2.28 \ - scikit-image=0.19 \ - scikit-learn=1.2 \ - scipy=1.10 \ - seaborn=0.12 \ - setuptools=65 \ - sphinx=4.5 \ - sphinx_rtd_theme=1.0 \ - svgutils=0.3 \ - toml=0.10 \ - traits=6.3 \ - zlib=1.2 \ - zstd=1.5; sync && \ - chmod -R a+rX ${CONDA_PATH}; sync && \ - chmod +x ${CONDA_PATH}/bin/*; sync && \ - ${CONDA_PATH}/bin/conda clean -afy && sync && \ - rm -rf ~/.conda ~/.cache/pip/*; sync +ENV MAMBA_ROOT_PREFIX="/opt/conda" +COPY env.yml /tmp/env.yml +COPY requirements.txt /tmp/requirements.txt +WORKDIR /tmp +RUN micromamba create -y -f /tmp/env.yml && \ + micromamba clean -y -a # Precaching fonts, set 'Agg' as default backend for matplotlib -RUN ${CONDA_PATH}/bin/python -c "from matplotlib import font_manager" && \ - sed -i 's/\(backend *: \).*$/\1Agg/g' $( ${CONDA_PATH}/bin/python -c "import matplotlib; print(matplotlib.matplotlib_fname())" ) +RUN micromamba -n base run python -c "from matplotlib import font_manager" && \ + sed -i 's/\(backend *: \).*$/\1Agg/g' $( micromamba -n base run python -c "import matplotlib; print(matplotlib.matplotlib_fname())" ) -# Install packages that are not distributed with conda -RUN ${CONDA_PATH}/bin/python -m pip install --no-cache-dir -U \ - datalad-osf \ - etelemetry \ - nitransforms \ - templateflow \ - transforms3d +# UV_USE_IO_URING for apparent race-condition (https://github.com/nodejs/node/issues/48444) +# Check if this is still necessary when updating the base image. +ENV PATH="/opt/conda/bin:$PATH" \ + UV_USE_IO_URING=0 # Installing SVGO and bids-validator -RUN ${CONDA_PATH}/bin/npm install -g svgo@^2.3 bids-validator@1.8.0 && \ +RUN npm install -g svgo@^3.2.0 bids-validator@^1.14.0 && \ rm -rf ~/.npm ~/.empty /root/.npm + +# Initialize templateflow +ENV TEMPLATEFLOW_HOME=/templateflow +RUN templateflow update + +# Pacify DataLad +RUN git config --global user.name "NiPreps Miniconda" \ + && git config --global user.email "nipreps@gmail.com" diff --git a/env.yml b/env.yml new file mode 100644 index 0000000..9c514b4 --- /dev/null +++ b/env.yml @@ -0,0 +1,31 @@ +name: base +channels: + - conda-forge +# Update this ~yearly; last updated Jan 2024 +dependencies: + - python=3.11 + # Needed for svgo and bids-validator; consider moving to deno + - nodejs=20 + # Intel Math Kernel Library for numpy + - mkl=2023.2.0 + - mkl-service=2.4.0 + # git-annex for templateflow users with DataLad superdatasets + - git-annex=*=alldep* + # Base scientific python stack; required by FSL, so pinned here + - numpy=1.26 + - scipy=1.11 + - matplotlib=3.8 + - pandas=2.2 + - h5py=3.10 + # Dependencies compiled against numpy, best to stick with conda + - nitime=0.10 + - scikit-image=0.22 + - scikit-learn=1.4 + # Utilities + - graphviz=9.0 + - pandoc=3.1 + - pip + - pip: + - -r requirements.txt +variables: + FSLOUTPUTTYPE: NIFTI_GZ \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f4e44ef --- /dev/null +++ b/requirements.txt @@ -0,0 +1,443 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --extra=container --strip-extras +# +annexremote==1.6.4 + # via + # datalad + # datalad-next + # datalad-osf +attrs==23.2.0 + # via + # niworkflows + # sdcflows +bids-validator==1.14.0 + # via pybids +boto==2.49.0 + # via datalad +certifi==2024.2.2 + # via + # requests + # sentry-sdk +cffi==1.16.0 + # via cryptography +chardet==5.2.0 + # via datalad +charset-normalizer==3.3.2 + # via requests +ci-info==0.3.0 + # via + # etelemetry + # migas +click==8.1.7 + # via + # codecarbon + # nipype + # pybids +contourpy==1.2.0 + # via + # bokeh + # matplotlib +cryptography==42.0.4 + # via secretstorage +cycler==0.12.1 + # via matplotlib +datalad==0.19.6 + # via + # datalad-next + # datalad-osf + # fmriprep (pyproject.toml) +datalad-next==1.2.0 + # via datalad-osf +datalad-osf==0.3.0 + # via fmriprep (pyproject.toml) +distro==1.9.0 + # via datalad +docopt==0.6.2 + # via num2words +etelemetry==0.3.1 + # via nipype +fasteners==0.19 + # via datalad +filelock==3.13.1 + # via nipype +fonttools==4.48.1 + # via matplotlib +formulaic==0.5.2 + # via pybids +greenlet==3.0.3 + # via sqlalchemy +h5py==3.10.0 + # via nitransforms +humanize==4.9.0 + # via + # datalad + # datalad-next +idna==3.6 + # via requests +imageio==2.33.1 + # via scikit-image +importlib-metadata==7.0.1 + # via keyring +importlib-resources==6.1.1 + # via + # nireports + # niworkflows + # templateflow +indexed-gzip==1.8.7 + # via smriprep +interface-meta==1.3.0 + # via formulaic +iso8601==2.1.0 + # via datalad +isodate==0.6.1 + # via rdflib +jaraco-classes==3.3.0 + # via + # keyring + # keyrings-alt +jeepney==0.8.0 + # via + # keyring + # secretstorage +jinja2==3.1.3 + # via + # bokeh + # niworkflows +joblib==1.3.2 + # via + # nilearn + # scikit-learn +keyring==24.3.0 + # via datalad +keyrings-alt==5.0.0 + # via datalad +kiwisolver==1.4.5 + # via matplotlib +lazy-loader==0.3 + # via scikit-image +lockfile==0.12.2 + # via smriprep +looseversion==1.3.0 + # via + # datalad + # fmriprep + # fmriprep (pyproject.toml) + # nipype + # niworkflows + # smriprep +lxml==5.1.0 + # via + # nilearn + # prov + # svgutils +markupsafe==2.1.5 + # via jinja2 +matplotlib==3.8.2 + # via + # nireports + # nitime + # niworkflows + # smriprep + # tedana +migas==0.4.0 + # via + # fmriprep + # sdcflows +more-itertools==10.2.0 + # via + # datalad-next + # jaraco-classes +msgpack==1.0.7 + # via datalad +networkx==3.2.1 + # via + # nipype + # prov + # scikit-image +nibabel==5.2.0 + # via + # fmriprep + # fmriprep (pyproject.toml) + # mapca + # nilearn + # nipype + # nireports + # nitransforms + # niworkflows + # pybids + # sdcflows + # smriprep + # tedana +nilearn==0.10.3 + # via + # mapca + # nireports + # niworkflows + # tedana +nipype==1.8.6 + # via + # fmriprep + # fmriprep (pyproject.toml) + # nireports + # niworkflows + # sdcflows + # smriprep +nireports==23.2.0 + # via + # fmriprep + # fmriprep (pyproject.toml) +nitime==0.10.2 + # via + # fmriprep + # fmriprep (pyproject.toml) +nitransforms==23.0.1 + # via + # fmriprep + # fmriprep (pyproject.toml) + # niworkflows + # sdcflows +niworkflows==1.10.0 + # via + # fmriprep + # fmriprep (pyproject.toml) + # sdcflows + # smriprep +num2words==0.5.13 + # via pybids +numpy==1.26.4 + # via + # bokeh + # contourpy + # fmriprep + # fmriprep (pyproject.toml) + # formulaic + # h5py + # imageio + # mapca + # matplotlib + # nibabel + # nilearn + # nipype + # nireports + # nitime + # nitransforms + # niworkflows + # pandas + # pybids + # scikit-image + # scikit-learn + # scipy + # sdcflows + # seaborn + # smriprep + # tedana + # tifffile +osfclient==0.0.5 + # via datalad-osf +packaging==23.2 + # via + # bokeh + # datalad + # etelemetry + # fmriprep + # fmriprep (pyproject.toml) + # matplotlib + # nibabel + # nilearn + # nipype + # niworkflows + # scikit-image + # smriprep +pandas==2.2.0 + # via + # bokeh + # codecarbon + # fmriprep + # fmriprep (pyproject.toml) + # formulaic + # nilearn + # nireports + # niworkflows + # pybids + # seaborn + # tedana +patool==2.1.1 + # via datalad +pillow==10.2.0 + # via + # bokeh + # imageio + # matplotlib + # scikit-image +platformdirs==4.2.0 + # via datalad +prometheus-client==0.19.0 + # via codecarbon +prov==2.0.0 + # via nipype +psutil==5.9.8 + # via + # codecarbon + # fmriprep + # fmriprep (pyproject.toml) +py-cpuinfo==9.0.0 + # via codecarbon +pybids==0.16.4 + # via + # fmriprep + # fmriprep (pyproject.toml) + # nireports + # niworkflows + # sdcflows + # smriprep + # templateflow +pycparser==2.21 + # via cffi +pydot==2.0.0 + # via nipype +pynvml==11.5.0 + # via codecarbon +pyparsing==3.1.1 + # via + # matplotlib + # pydot + # rdflib +python-dateutil==2.8.2 + # via + # arrow + # matplotlib + # nipype + # pandas + # prov +python-gitlab==4.4.0 + # via datalad +pytz==2024.1 + # via + # apscheduler + # pandas +pyyaml==6.0.1 + # via + # bokeh + # nireports + # niworkflows + # smriprep +rapidfuzz==3.6.1 + # via codecarbon +rdflib==7.0.0 + # via + # nipype + # prov +requests==2.31.0 + # via + # codecarbon + # datalad + # etelemetry + # fmriprep + # fmriprep (pyproject.toml) + # nilearn + # osfclient + # python-gitlab + # requests-toolbelt + # templateflow +requests-toolbelt==1.0.0 + # via python-gitlab +scikit-image==0.22.0 + # via + # niworkflows + # sdcflows +scikit-learn==1.4.0 + # via + # mapca + # nilearn + # tedana +scipy==1.11.4 + # via + # formulaic + # mapca + # nilearn + # nipype + # nitime + # nitransforms + # niworkflows + # pybids + # scikit-image + # scikit-learn + # sdcflows + # tedana +sdcflows==2.8.0 + # via + # fmriprep + # fmriprep (pyproject.toml) +secretstorage==3.3.3 + # via keyring +simplejson==3.19.2 + # via nipype +six==1.16.0 + # via + # apscheduler + # isodate + # osfclient + # python-dateutil +sqlalchemy==2.0.25 + # via pybids +svgutils==0.3.4 + # via + # nireports + # niworkflows +templateflow==24.2.0 + # via + # fmriprep + # fmriprep (pyproject.toml) + # nireports + # niworkflows + # sdcflows + # smriprep +threadpoolctl==3.2.0 + # via + # scikit-learn + # tedana +tifffile==2024.1.30 + # via scikit-image +toml==0.10.2 + # via + # fmriprep + # fmriprep (pyproject.toml) + # sdcflows +tornado==6.4 + # via bokeh +tqdm==4.66.1 + # via + # datalad + # osfclient + # templateflow +traits==6.3.2 + # via + # nipype + # niworkflows + # sdcflows +transforms3d==0.4.1 + # via niworkflows +types-python-dateutil==2.8.19.20240106 + # via arrow +typing-extensions==4.9.0 + # via + # datalad + # formulaic + # sqlalchemy +tzdata==2023.4 + # via pandas +tzlocal==5.2 + # via apscheduler +urllib3==2.2.0 + # via + # requests + # sentry-sdk +wrapt==1.16.0 + # via formulaic +xyzservices==2023.10.1 + # via bokeh +zipp==3.17.0 + # via importlib-metadata \ No newline at end of file