From 26992a0a162c1370a85874c689f9332ad6020a98 Mon Sep 17 00:00:00 2001 From: Tyler Chafin Date: Tue, 17 Sep 2024 10:31:07 +0100 Subject: [PATCH 01/15] removed accidentally committed file --- seq_cache_populate.pl | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 seq_cache_populate.pl diff --git a/seq_cache_populate.pl b/seq_cache_populate.pl deleted file mode 100644 index e69de29..0000000 From 93a680efcdb88bf1b57edf73d46c8b177e0ecff5 Mon Sep 17 00:00:00 2001 From: Tyler Chafin Date: Tue, 17 Sep 2024 10:34:29 +0100 Subject: [PATCH 02/15] add alternate mappers to CI --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3051872..cc6eddc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,9 @@ jobs: NXF_VER: - "22.10.1" - "latest-everything" + short_aligner: + - bwamem2 + - minimap2 steps: - name: Check out pipeline code uses: actions/checkout@v3 @@ -35,9 +38,9 @@ jobs: with: version: "${{ matrix.NXF_VER }}" - - name: Run pipeline with test data + - name: Run pipeline with test data and bwa-mem2 # You can customise CI pipeline run tests as required # For example: adding multiple test runs with different parameters # Remember that you can parallelise this by using strategy.matrix run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results --short_aligner ${{ matrix.short_aligner }} From e937591c81763fc4452040218dc9e59373b86401 Mon Sep 17 00:00:00 2001 From: Tyler Chafin Date: Tue, 17 Sep 2024 10:39:32 +0100 Subject: [PATCH 03/15] update docs to note interleaved fastq --- assets/schema_input.json | 2 +- docs/usage.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/schema_input.json b/assets/schema_input.json index c1197d7..22636f3 100644 --- a/assets/schema_input.json +++ b/assets/schema_input.json @@ -22,7 +22,7 @@ "datafile": { "format": "string", "pattern": "^\\S+$", - "errorMessage": "Data file for reads cannot contain spaces and must have extension 'cram', 'bam', '.fq.gz' or '.fastq.gz'", + "errorMessage": "Data file for reads cannot contain spaces and must have extension 'cram', 'bam', '.fq.gz' or '.fastq.gz'. Note that FASTQ inputs should be interleaved if paired-end.", "meta": ["datafile"] }, "library": { diff --git a/docs/usage.md b/docs/usage.md index 9423325..75772a0 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -42,7 +42,7 @@ sample1_T5,pacbio,pacbio2.bam,pacbio2 | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `sample` | Custom sample name. This entry will be identical for multiple sequencing libraries/runs from the same sample. Spaces in sample names are automatically converted to underscores (\_). | | `datatype` | Type of sequencing data. Must be one of `hic`, `Illumina`, `pacbio`, or `ont`. | -| `datafile` | Full path to read data file. Must be `bam`, `cram`, `fastq.gz` or `fq.gz` for `Illumina` and `HiC`. Must be `bam`, `fastq.gz` or `fq.gz` for `pacbio`. Must be `fastq.gz` or `fq.gz` for `ont`. | +| `datafile` | Data file for reads cannot contain spaces and must have extension 'cram', 'bam', '.fq.gz' or '.fastq.gz'. Note that FASTQ inputs should be interleaved if paired-end. | | `library` | (Optional) The library value is a unique identifier which is assigned to read group (`@RG`) ID. If the library name is not specified, the pipeline will auto-create library name using the data filename provided in the samplesheet. | An [example samplesheet](../assets/samplesheet.csv) has been provided with the pipeline. From 975d48a1db5ae3a2866831afdf5e915ffca56ea3 Mon Sep 17 00:00:00 2001 From: Tyler Chafin Date: Tue, 17 Sep 2024 10:39:40 +0100 Subject: [PATCH 04/15] update copyright --- LICENSE | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index e238724..257404f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,8 @@ MIT License -Copyright (c) @priyanka-surana +Copyright (c) 2022-2024 Genome Research Ltd. +except `bin/filter_five_end.pl`: +Copyright (c) 2017 Arima Genomics, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From e3d859a38983d93bb6a9878388b11387ddfd34e9 Mon Sep 17 00:00:00 2001 From: Tyler Chafin Date: Tue, 17 Sep 2024 10:42:45 +0100 Subject: [PATCH 05/15] prettier linting --- docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 75772a0..50b0a22 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -42,7 +42,7 @@ sample1_T5,pacbio,pacbio2.bam,pacbio2 | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `sample` | Custom sample name. This entry will be identical for multiple sequencing libraries/runs from the same sample. Spaces in sample names are automatically converted to underscores (\_). | | `datatype` | Type of sequencing data. Must be one of `hic`, `Illumina`, `pacbio`, or `ont`. | -| `datafile` | Data file for reads cannot contain spaces and must have extension 'cram', 'bam', '.fq.gz' or '.fastq.gz'. Note that FASTQ inputs should be interleaved if paired-end. | +| `datafile` | Data file for reads cannot contain spaces and must have extension 'cram', 'bam', '.fq.gz' or '.fastq.gz'. Note that FASTQ inputs should be interleaved if paired-end. | | `library` | (Optional) The library value is a unique identifier which is assigned to read group (`@RG`) ID. If the library name is not specified, the pipeline will auto-create library name using the data filename provided in the samplesheet. | An [example samplesheet](../assets/samplesheet.csv) has been provided with the pipeline. From 571b137a6f6216714de9852ab7e8ef6f7cb090da Mon Sep 17 00:00:00 2001 From: Tyler Chafin Date: Tue, 24 Sep 2024 12:32:54 +0100 Subject: [PATCH 06/15] Update download_pipeline.yml --- .github/workflows/download_pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml index bd9f7bf..36fbc9d 100644 --- a/.github/workflows/download_pipeline.yml +++ b/.github/workflows/download_pipeline.yml @@ -10,7 +10,7 @@ on: testbranch: description: "The specific branch you wish to utilize for the test execution of nf-core download." required: true - default: "dev" + default: "main" pull_request: types: - opened From abee7626de102e432f37608d5b506b66ceef0b9d Mon Sep 17 00:00:00 2001 From: Tyler Chafin Date: Tue, 24 Sep 2024 12:39:44 +0100 Subject: [PATCH 07/15] Update linting.yml --- .github/workflows/linting.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 177172b..4392dc3 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -22,7 +22,13 @@ jobs: run: npm install -g editorconfig-checker - name: Run ECLint check - run: editorconfig-checker -exclude README.md $(find .* -type f | grep -v '.git\|.py\|.md\|json\|yml\|yaml\|html\|css\|work\|.nextflow\|build\|nf_core.egg-info\|log.txt\|Makefile') + run: | + files=$(find .* -type f | grep -v '.git\|.py\|.md\|json\|yml\|yaml\|html\|css\|work\|.nextflow\|build\|nf_core.egg-info\|log.txt\|Makefile') + if [ -n "$files" ]; then + editorconfig-checker -exclude README.md $files + else + echo "No files found to check." + fi Prettier: runs-on: ubuntu-latest From dbaad3de1ed845d53a6b86009447556f0e6706c8 Mon Sep 17 00:00:00 2001 From: Tyler Chafin Date: Tue, 24 Sep 2024 12:43:16 +0100 Subject: [PATCH 08/15] Update linting.yml --- .github/workflows/linting.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 4392dc3..1b3803f 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -19,16 +19,10 @@ jobs: - uses: actions/setup-node@v3 - name: Install editorconfig-checker - run: npm install -g editorconfig-checker + run: npm install editorconfig-checker - name: Run ECLint check - run: | - files=$(find .* -type f | grep -v '.git\|.py\|.md\|json\|yml\|yaml\|html\|css\|work\|.nextflow\|build\|nf_core.egg-info\|log.txt\|Makefile') - if [ -n "$files" ]; then - editorconfig-checker -exclude README.md $files - else - echo "No files found to check." - fi + run: editorconfig-checker -exclude README.md $(find .* -type f | grep -v '.git\|.py\|.md\|json\|yml\|yaml\|html\|css\|work\|.nextflow\|build\|nf_core.egg-info\|log.txt\|Makefile') Prettier: runs-on: ubuntu-latest From e10c89a6929cecfa9f8d0b8ea307adb3da02a2b5 Mon Sep 17 00:00:00 2001 From: Tyler Chafin Date: Tue, 24 Sep 2024 12:45:07 +0100 Subject: [PATCH 09/15] Update linting.yml --- .github/workflows/linting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 1b3803f..177172b 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-node@v3 - name: Install editorconfig-checker - run: npm install editorconfig-checker + run: npm install -g editorconfig-checker - name: Run ECLint check run: editorconfig-checker -exclude README.md $(find .* -type f | grep -v '.git\|.py\|.md\|json\|yml\|yaml\|html\|css\|work\|.nextflow\|build\|nf_core.egg-info\|log.txt\|Makefile') From 7aa563d397a9d6799f5268321613aea4e3c1905a Mon Sep 17 00:00:00 2001 From: Tyler Chafin Date: Tue, 24 Sep 2024 12:53:21 +0100 Subject: [PATCH 10/15] Update linting.yml --- .github/workflows/linting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 177172b..19ddb83 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-node@v3 - name: Install editorconfig-checker - run: npm install -g editorconfig-checker + run: npm install -g editorconfig-checker@3.0.2 - name: Run ECLint check run: editorconfig-checker -exclude README.md $(find .* -type f | grep -v '.git\|.py\|.md\|json\|yml\|yaml\|html\|css\|work\|.nextflow\|build\|nf_core.egg-info\|log.txt\|Makefile') From fd476b90923a8f9c84c4305139eb38825946a610 Mon Sep 17 00:00:00 2001 From: Tyler Chafin Date: Tue, 24 Sep 2024 13:04:37 +0100 Subject: [PATCH 11/15] Update ci.yml --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc6eddc..10d6e95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ name: nf-core CI -# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors + on: push: branches: @@ -18,10 +18,10 @@ concurrency: jobs: test: name: Run pipeline with test data - # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'sanger-tol/readmapping') }}" runs-on: ubuntu-latest strategy: + # Use the 'fail-fast: false' option to allow all combinations to run + fail-fast: false matrix: NXF_VER: - "22.10.1" @@ -38,9 +38,9 @@ jobs: with: version: "${{ matrix.NXF_VER }}" - - name: Run pipeline with test data and bwa-mem2 - # You can customise CI pipeline run tests as required - # For example: adding multiple test runs with different parameters - # Remember that you can parallelise this by using strategy.matrix + - name: Run pipeline with test data run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results --short_aligner ${{ matrix.short_aligner }} + nextflow run ${GITHUB_WORKSPACE} \ + -profile test,docker \ + --outdir ./results \ + --short_aligner ${{ matrix.short_aligner }} From ba7778bb4a1f4d743a09c168eb802b91797c333f Mon Sep 17 00:00:00 2001 From: Tyler Chafin Date: Tue, 24 Sep 2024 13:11:13 +0100 Subject: [PATCH 12/15] Update download_pipeline.yml --- .github/workflows/download_pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml index 36fbc9d..9ea6412 100644 --- a/.github/workflows/download_pipeline.yml +++ b/.github/workflows/download_pipeline.yml @@ -43,7 +43,7 @@ jobs: architecture: "x64" - uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7 with: - singularity-version: 3.8.3 + singularity-version: 3.8.7 - name: Install dependencies run: | From 5b2dd61716f7fd87186c550905e3f49efe22afe6 Mon Sep 17 00:00:00 2001 From: Tyler Chafin Date: Tue, 24 Sep 2024 13:12:27 +0100 Subject: [PATCH 13/15] Update ci.yml --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10d6e95..cc6eddc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ name: nf-core CI - +# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors on: push: branches: @@ -18,10 +18,10 @@ concurrency: jobs: test: name: Run pipeline with test data + # Only run on push if this is the nf-core dev branch (merged PRs) + if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'sanger-tol/readmapping') }}" runs-on: ubuntu-latest strategy: - # Use the 'fail-fast: false' option to allow all combinations to run - fail-fast: false matrix: NXF_VER: - "22.10.1" @@ -38,9 +38,9 @@ jobs: with: version: "${{ matrix.NXF_VER }}" - - name: Run pipeline with test data + - name: Run pipeline with test data and bwa-mem2 + # You can customise CI pipeline run tests as required + # For example: adding multiple test runs with different parameters + # Remember that you can parallelise this by using strategy.matrix run: | - nextflow run ${GITHUB_WORKSPACE} \ - -profile test,docker \ - --outdir ./results \ - --short_aligner ${{ matrix.short_aligner }} + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results --short_aligner ${{ matrix.short_aligner }} From d3855bc28d0e1aea4ff90a5e1f41621903c71a2a Mon Sep 17 00:00:00 2001 From: Tyler Chafin Date: Tue, 24 Sep 2024 13:18:14 +0100 Subject: [PATCH 14/15] Update ci.yml --- .github/workflows/ci.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc6eddc..40fb2e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ name: nf-core CI -# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors + on: push: branches: @@ -18,10 +18,9 @@ concurrency: jobs: test: name: Run pipeline with test data - # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'sanger-tol/readmapping') }}" runs-on: ubuntu-latest strategy: + fail-fast: false matrix: NXF_VER: - "22.10.1" @@ -38,9 +37,6 @@ jobs: with: version: "${{ matrix.NXF_VER }}" - - name: Run pipeline with test data and bwa-mem2 - # You can customise CI pipeline run tests as required - # For example: adding multiple test runs with different parameters - # Remember that you can parallelise this by using strategy.matrix + - name: Run pipeline with test data run: | nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results --short_aligner ${{ matrix.short_aligner }} From c5d53f8bb3c91ea23286066e87907684de1be196 Mon Sep 17 00:00:00 2001 From: Tyler Chafin Date: Tue, 24 Sep 2024 13:22:52 +0100 Subject: [PATCH 15/15] Delete .github/workflows/download_pipeline.yml --- .github/workflows/download_pipeline.yml | 88 ------------------------- 1 file changed, 88 deletions(-) delete mode 100644 .github/workflows/download_pipeline.yml diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml deleted file mode 100644 index 9ea6412..0000000 --- a/.github/workflows/download_pipeline.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Test successful pipeline download with 'nf-core download' - -# Run the workflow when: -# - dispatched manually -# - when a PR is opened or reopened to master branch -# - the head branch of the pull request is updated, i.e. if fixes for a release are pushed last minute to dev. -on: - workflow_dispatch: - inputs: - testbranch: - description: "The specific branch you wish to utilize for the test execution of nf-core download." - required: true - default: "main" - pull_request: - types: - - opened - - edited - - synchronize - branches: - - main - - dev - pull_request_target: - branches: - - main - - dev - -env: - NXF_ANSI_LOG: false - -jobs: - download: - runs-on: ubuntu-latest - steps: - - name: Install Nextflow - uses: nf-core/setup-nextflow@v2 - - - name: Disk space cleanup - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 - with: - python-version: "3.12" - architecture: "x64" - - uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7 - with: - singularity-version: 3.8.7 - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install git+https://github.com/nf-core/tools.git@dev - - - name: Get the repository name and current branch set as environment variable - run: | - echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> ${GITHUB_ENV} - echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> ${GITHUB_ENV} - echo "REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> ${GITHUB_ENV} - - - name: Download the pipeline - env: - NXF_SINGULARITY_CACHEDIR: ./ - run: | - nf-core download ${{ env.REPO_LOWERCASE }} \ - --revision ${{ env.REPO_BRANCH }} \ - --outdir ./${{ env.REPOTITLE_LOWERCASE }} \ - --compress "none" \ - --container-system 'singularity' \ - --container-library "quay.io" -l "docker.io" -l "ghcr.io" \ - --container-cache-utilisation 'amend' \ - --download-configuration - - - name: Inspect download - run: tree ./${{ env.REPOTITLE_LOWERCASE }} - - - name: Run the downloaded pipeline (stub) - id: stub_run_pipeline - continue-on-error: true - env: - NXF_SINGULARITY_CACHEDIR: ./ - NXF_SINGULARITY_HOME_MOUNT: true - run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results - - name: Run the downloaded pipeline (stub run not supported) - id: run_pipeline - if: ${{ job.steps.stub_run_pipeline.status == failure() }} - env: - NXF_SINGULARITY_CACHEDIR: ./ - NXF_SINGULARITY_HOME_MOUNT: true - run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -profile test,singularity --outdir ./results