Merge pull request #2440 from nf-core/dev

Dev -> Master for v2.10

.devcontainer/devcontainer.json

@@ -3,6 +3,7 @@
     "image": "nfcore/gitpod:latest",
     "postCreateCommand": "python -m pip install --upgrade -r ../requirements-dev.txt -e ../ && pre-commit install --install-hooks",
     "remoteUser": "gitpod",
+    "runArgs": ["--privileged"],
 
     // Configure tool-specific properties.
     "customizations": {

.github/RELEASE_CHECKLIST.md

@@ -3,16 +3,17 @@
 1. Check issue milestones to see outstanding issues to resolve if possible or transfer to the milestones for the next release e.g. [`v1.9`](https://github.com/nf-core/tools/issues?q=is%3Aopen+is%3Aissue+milestone%3A1.9)
 2. Most importantly, pick an undeniably outstanding [name](http://www.codenamegenerator.com/) for the release where _Prefix_ = _Metal_ and _Dictionary_ = _Animal_.
 3. Check the [pipeline health page](https://nf-co.re/pipeline_health) to make sure that all repos look sane (missing `TEMPLATE` branches etc)
-4. Create a PR to `dev` to bump the version in `CHANGELOG.md` and `setup.py` and change the gitpod container to `nfcore/gitpod:latest`.
-5. Make sure all CI tests are passing!
-6. Create a PR from `dev` to `master`
-7. Make sure all CI tests are passing again (additional tests are run on PRs to `master`)
-8. Request review (2 approvals required)
-9. Run `rich-codex` to regenerate docs screengrabs (actions `workflow_dispatch` button)
-10. Merge the PR into `master`
-11. Wait for CI tests on the commit to passed
-12. (Optional but a good idea) Run a manual sync on `nf-core/testpipeline` and check that CI is passing on the resulting PR.
-13. Create a new release copying the `CHANGELOG` for that release into the description section.
+4. Check that modules/subworkflows in tempalte are up to date with the latest releases
+5. Create a PR to `dev` to bump the version in `CHANGELOG.md` and `setup.py` and change the gitpod container to `nfcore/gitpod:latest`.
+6. Make sure all CI tests are passing!
+7. Create a PR from `dev` to `master`
+8. Make sure all CI tests are passing again (additional tests are run on PRs to `master`)
+9. Request review (2 approvals required)
+10. Run `rich-codex` to regenerate docs screengrabs (actions `workflow_dispatch` button)
+11. Merge the PR into `master`
+12. Wait for CI tests on the commit to passed
+13. (Optional but a good idea) Run a manual sync on `nf-core/testpipeline` and check that CI is passing on the resulting PR.
+14. Create a new release copying the `CHANGELOG` for that release into the description section.
 
 ## After release
 

.github/workflows/create-lint-wf.yml

@@ -31,10 +31,10 @@ jobs:
         name: Check out source-code repository
 
       # Set up nf-core/tools
-      - name: Set up Python 3.8
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.11
 
       - name: Install python dependencies
         run: |

.github/workflows/create-test-lint-wf-template.yml

@@ -35,10 +35,10 @@ jobs:
       - uses: actions/checkout@v3
         name: Check out source-code repository
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.11
 
       - name: Install python dependencies
         run: |
@@ -97,7 +97,7 @@ jobs:
 
       # Try syncing it before we change anything
       - name: nf-core sync
-        run: nf-core --log-file log.txt sync --dir my-prefix-testpipeline/ --template-yaml ${{ matrix.TEMPLATE }}
+        run: nf-core --log-file log.txt sync --dir my-prefix-testpipeline/
 
       # Run code style linting
       - name: Run Prettier --check

.github/workflows/create-test-wf.yml

@@ -29,10 +29,10 @@ jobs:
       - uses: actions/checkout@v3
         name: Check out source-code repository
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.11
 
       - name: Install python dependencies
         run: |

.github/workflows/deploy-pypi.yml

@@ -16,10 +16,10 @@ jobs:
       - uses: actions/checkout@v3
         name: Check out source-code repository
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.11
 
       - name: Install python dependencies
         run: |

.github/workflows/fix-linting.yml

@@ -38,10 +38,10 @@ jobs:
           # Override to remove the default --check flag so that we make changes
           options: "--color"
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.11
       - name: python-isort
         uses: isort/[email protected]
         with:

.github/workflows/lint-code.yml

@@ -76,10 +76,10 @@ jobs:
       - name: Check out source-code repository
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.11
       - name: python-isort
         uses: isort/[email protected]
         with:

.github/workflows/pytest.yml

@@ -5,7 +5,13 @@ on:
   push:
     branches:
       - dev
+    paths-ignore:
+      - "docs/**"
+      - "CHANGELOG.md"
   pull_request:
+    paths-ignore:
+      - "docs/**"
+      - "CHANGELOG.md"
   release:
     types: [published]
 
@@ -14,12 +20,19 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  GITHUB_TOKEN: ${{ github.token }}
+
 jobs:
   pytest:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runner }}
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.11"]
+        runner: ["ubuntu-latest"]
+        include:
+          - runner: "ubuntu-20.04"
+            python-version: "3.8"
 
     steps:
       - uses: actions/checkout@v3
@@ -35,6 +48,14 @@ jobs:
           python -m pip install --upgrade pip -r requirements-dev.txt
           pip install -e .
 
+      - name: Downgrade git to the Ubuntu official repository's version
+        if: ${{ matrix.runner == 'ubuntu-20.04' && matrix.python-version == '3.8' }}
+        run: |
+          sudo apt update
+          sudo apt remove git git-man
+          sudo add-apt-repository --remove ppa:git-core/ppa
+          sudo apt install git
+
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v1
         with:

.github/workflows/sync.yml

@@ -48,10 +48,10 @@ jobs:
           path: nf-core/${{ matrix.pipeline }}
           fetch-depth: "0"
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.11
 
       - name: Install python dependencies
         run: |

.github/workflows/tools-api-docs-dev.yml

@@ -4,7 +4,11 @@ on:
   push:
     branches:
       - dev
+    paths-ignore:
+      - "CHANGELOG.md"
   pull_request:
+    paths-ignore:
+      - "CHANGELOG.md"
   release:
     types: [published]
 
@@ -22,10 +26,10 @@ jobs:
       - name: Check out source-code repository
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.11
 
       - name: Install python dependencies
         run: |
@@ -45,6 +49,6 @@ jobs:
           username: ${{ secrets.ftp_username}}
           password: ${{ secrets.ftp_password }}
           local-dir: "./docs/api/_build/html/"
-          server-dir: ${{ secrets.ftp_server_dir }}/dev/
+          server-dir: ${{ secrets.ftp_server_old_site_dir }}/dev/
           protocol: ${{ secrets.ftp_protocol }}
           port: ${{ secrets.ftp_port }}

.github/workflows/tools-api-docs-release.yml

@@ -21,10 +21,10 @@ jobs:
       - name: Check out source-code repository
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.11
 
       - name: Install python dependencies
         run: |
@@ -43,6 +43,6 @@ jobs:
           username: ${{ secrets.ftp_username}}
           password: ${{ secrets.ftp_password }}
           local-dir: "./docs/api/_build/html/"
-          server-dir: ${{ secrets.ftp_server_dir }}/${{ matrix.dir }}/
+          server-dir: ${{ secrets.ftp_server_old_site_dir }}/${{ matrix.dir }}/
           protocol: ${{ secrets.ftp_protocol }}
           port: ${{ secrets.ftp_port }}

CHANGELOG.md

@@ -1,13 +1,57 @@
 # nf-core/tools: Changelog
 
+# [v2.10 - Nickel Ostrich](https://github.com/nf-core/tools/releases/tag/2.10) + [2023-09-25]
+
+### Template
+
+- Fix links in `multiqc_config.yml` ([#2372](https://github.com/nf-core/tools/pull/2372) and [#2412](https://github.com/nf-core/tools/pull/2412))
+- Remove default false from nextflow_schema.json ([#2376](https://github.com/nf-core/tools/pull/2376))
+- Add module MULTIQC to modules.config ([#2377](https://github.com/nf-core/tools/pull/2377))
+- Add GitHub workflow for automated release announcements ([#2382](https://github.com/nf-core/tools/pull/2382))
+- Update the Code of Conduct ([#2381](https://github.com/nf-core/tools/pull/2381))
+- Save template information to `.nf-core.yml` and deprecate argument `--template-yaml` for `nf-core sync` ([#2388](https://github.com/nf-core/tools/pull/2388) and [#2389](https://github.com/nf-core/tools/pull/2389))
+- ([#2397](https://github.com/nf-core/tools/pull/2397)) Remove fixed Ubuntu test and added to standard testing matrix
+- ([#2396](https://github.com/nf-core/tools/pull/2396)) Reduce container finding error to warning since the registries are not consistent.
+- ([#2415](https://github.com/nf-core/tools/pull/2415#issuecomment-1709847086)) Add autoMounts for apptainer.
+- Remove `igenomes_base` from the schema, so that nf-validation doesn't create a file path and throw errors offline for s3 objects.
+- Modified devcontainer permissions so that singularity can be run in Codespaces/VS Code devcontainers ([Commit a103f44](https://github.com/CarsonJM/tools/commit/a103f4484eca8c6d668e4653a4ed8d20faf1b41d))
+- Update Gitpod profile resources to reflect base environment settings.
+- ([#747](https://github.com/nf-core/tools/issues/747)) Add to the template the code to dump the selected pipeline parameters into a json file.
+
+### Download
+
+- Improved container image resolution and prioritization of http downloads over Docker URIs ([#2364](https://github.com/nf-core/tools/pull/2364)).
+- Registries provided with `-l`/`--container-library` will be ignored for modules with explicit container registry specifications ([#2403](https://github.com/nf-core/tools/pull/2403)).
+- Fix unintentional downloading of containers in test for the Tower download functionality. Bug reported by @adamrtalbot and @awgymer ([#2434](https://github.com/nf-core/tools/pull/2434)).
+
+### Linting
+
+- Add new command `nf-core subworkflows lint` ([#2379](https://github.com/nf-core/tools/pull/2379))
+
+### Modules
+
+### Subworkflows
+
+- Fix bug: missing subworkflow name when using `nf-core subworkflows create` ([#2435](https://github.com/nf-core/tools/pull/2435))
+
+### General
+
+- Initialise `docker_image_name` to fix `UnboundLocalError` error ([#2374](https://github.com/nf-core/tools/pull/2374))
+- Fix prompt pipeline revision during launch ([#2375](https://github.com/nf-core/tools/pull/2375))
+- Add a `create-params-file` command to create a YAML parameter file for a pipeline containing parameter documentation and defaults. ([#2362](https://github.com/nf-core/tools/pull/2362))
+- Update the Code of Conduct ([#2381](https://github.com/nf-core/tools/pull/2381))
+- Remove `--no-git` option from `nf-core create` ([#2394](https://github.com/nf-core/tools/pull/2394))
+- Throw warning when custom workflow name contains special characters ([#2401](https://github.com/nf-core/tools/pull/2401))
+- Bump version of nf-test snapshot files with `nf-core bump-version` ([#2410](https://github.com/nf-core/tools/pull/2410))
+
 # [v2.9 - Chromium Falcon](https://github.com/nf-core/tools/releases/tag/2.9) + [2023-06-29]
 
 ### Template
 
 - `params.max_multiqc_email_size` is no longer required ([#2273](https://github.com/nf-core/tools/pull/2273))
 - Remove `cleanup = true` from `test_full.config` in pipeline template ([#2279](https://github.com/nf-core/tools/pull/2279))
 - Fix usage docs for specifying `params.yaml` ([#2279](https://github.com/nf-core/tools/pull/2279))
-- Added stub in modules template ([#2277])(https://github.com/nf-core/tools/pull/2277) [Contributed by @nvnieuwk]
+- Added stub in modules template ([#2277](https://github.com/nf-core/tools/pull/2277)) [Contributed by @nvnieuwk]
 - Move registry definitions out of profile scope ([#2286])(https://github.com/nf-core/tools/pull/2286)
 - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287)
 - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291))
@@ -50,7 +94,7 @@ _In addition, `-r` / `--revision` has been changed to a parameter that can be pr
 ### Linting
 
 - Warn if container access is denied ([#2270](https://github.com/nf-core/tools/pull/2270))
-- Error if module container specification has quay.io as prefix when it shouldn't have ([#2278])(https://github.com/nf-core/tools/pull/2278/files)
+- Error if module container specification has quay.io as prefix when it shouldn't have ([#2278](https://github.com/nf-core/tools/pull/2278/files)
 - Detect if container is 'simple name' and try to contact quay.io server by default ([#2281](https://github.com/nf-core/tools/pull/2281))
 - Warn about null/None/empty default values in `nextflow_schema.json` ([#3328](https://github.com/nf-core/tools/pull/2328))
 - Fix linting when creating a pipeline skipping some parts of the template and add CI test ([#2330](https://github.com/nf-core/tools/pull/2330))
@@ -81,7 +125,7 @@ _In addition, `-r` / `--revision` has been changed to a parameter that can be pr
 - Consistent syntax for branch checks in PRs ([#2202](https://github.com/nf-core/tools/issues/2202))
 - Fixed minor Jinja2 templating bug that caused the PR template to miss a newline
 - Updated AWS tests to use newly moved `seqeralabs/action-tower-launch` instead of `nf-core/tower-action`
-- Remove `.cff` files from `.editorconfig` [(#2145)[https://github.com/nf-core/tools/pull/2145]]
+- Remove `.cff` files from `.editorconfig` ([#2145](https://github.com/nf-core/tools/pull/2145))
 - Simplify pipeline README ([#2186](https://github.com/nf-core/tools/issues/2186))
 - Added support for the apptainer container engine via `-profile apptainer`. ([#2244](https://github.com/nf-core/tools/issues/2244)) [Contributed by @jfy133]
 - Added config `docker.registry` to pipeline template for a configurable default container registry when using Docker containers. Defaults to `quay.io` ([#2133](https://github.com/nf-core/tools/pull/2133))
@@ -1108,7 +1152,7 @@ making a pull-request. See [`.github/CONTRIBUTING.md`](.github/CONTRIBUTING.md)
 - Move `params`section from `base.config` to `nextflow.config`
 - Use `env` scope to export `PYTHONNOUSERSITE` in `nextflow.config` to prevent conflicts with host Python environment
 - Bump minimum Nextflow version to `19.10.0` - required to properly use `env` scope in `nextflow.config`
-- Added support for nf-tower in the travis tests, using public mailbox [email protected]
+- Added support for nf-tower in the travis tests, using public mailbox <[email protected]>
 - Add link to [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and [Semantic Versioning](http://semver.org/spec/v2.0.0.html) to CHANGELOG
 - Adjusted `.travis.yml` checks to allow for `patch` branches to be tested
 - Add Python 3.7 dependency to the `environment.yml` file