Merge branch 'master' into fgbio-read-clipping

IKIM-Essen · Nov 22, 2023 · 903691f · 903691f
2 parents 2429757 + b232c00
commit 903691f
Show file tree

Hide file tree

Showing 74 changed files with 473 additions and 398 deletions.
diff --git a/.github/workflows/check-todos.yml b/.github/workflows/check-todos.yml
@@ -11,4 +11,4 @@ jobs:
     steps:
       - uses: "actions/checkout@master"
       - name: "TODO to Issue"
-        uses: "alstr/todo-to-issue-action@v4.6"
+        uses: "alstr/todo-to-issue-action@v4.11"
diff --git a/.github/workflows/conventional-prs.yml b/.github/workflows/conventional-prs.yml
@@ -12,6 +12,6 @@ jobs:
   title-format:
     runs-on: ubuntu-latest
     steps:
-      - uses: amannn/action-semantic-pull-request@v4
+      - uses: amannn/action-semantic-pull-request@v5
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/copyright-preamble.yml b/.github/workflows/copyright-preamble.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       # Checking out the repository which is to be actioned
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       # Implementing action on repository
       - uses: VinnyBabuManjaly/[email protected]
         with:
@@ -25,7 +25,7 @@ jobs:
 
       # Creates pull request with all changes in file
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v4
+        uses: peter-evans/create-pull-request@v5
         with:
           commit-message: "chore: update copyright preamble"
           delete-branch: true

diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -18,10 +18,10 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: SetUp Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: 3.x
 

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.ref != 'refs/heads/master'
     steps:
-      - uses: khan/[email protected].0
+      - uses: khan/[email protected].1
         with:
           workflows: "main.yml"
         env:
@@ -22,9 +22,9 @@ jobs:
   Formatting:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Formatting
-        uses: github/super-linter@v4.9.2
+        uses: github/super-linter@v5.0.0
         env:
           VALIDATE_ALL_CODEBASE: false
           DEFAULT_BRANCH: master
@@ -39,22 +39,25 @@ jobs:
     env:
       GISAID_API_TOKEN: ${{ secrets.GISAID_API_TOKEN }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Lint workflow
-        uses: snakemake/snakemake-github-action@v1.24.0
+        uses: snakemake/snakemake-github-action@v1.25.1
         with:
           directory: .
           snakefile: workflow/Snakefile
           stagein: mamba install -n snakemake -c conda-forge peppy
           args: "--lint"
 
-  Pre-Commit:
-    runs-on: ubuntu-latest
-    if: github.ref != 'refs/heads/master'
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v3
-      - uses: pre-commit/[email protected]
+  # pre-commit action currently fails:
+  # https://github.com/IKIM-Essen/uncovar/actions/runs/4304753941/jobs/7506225198#step:4:115
+  # revisit when new pre-commit release >3.0.0 is out
+  # Pre-Commit:
+  #   runs-on: ubuntu-latest
+  #   if: github.ref != 'refs/heads/master'
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: actions/setup-python@v4
+  #     - uses: pre-commit/[email protected]
 
   Technology-Tests:
     runs-on: ubuntu-latest
@@ -63,14 +66,19 @@ jobs:
     needs:
       - Formatting
       - Linting
-      - Pre-Commit
+      #- Pre-Commit
     strategy:
       matrix:
         rule: [all, all -npr]
+        # disable ont actions
         technology: [all, illumina, ont, ion]
+        # technology: [all, illumina, ion]
         seq_method: [shotgun, amplicon]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
 
       # android - will release about 10 GB if you don't need Android
       # dotnet  - will release about 20 GB if you don't need .NET
@@ -85,11 +93,9 @@ jobs:
           mkdir -p .tests/data
           curl -L https://github.com/thomasbtf/small-kraken-db/raw/master/B.1.1.7.reads.1.fastq.gz > .tests/data/B117.1.fastq.gz
           curl -L https://github.com/thomasbtf/small-kraken-db/raw/master/B.1.1.7.reads.1.fastq.gz > .tests/data/B117.2.fastq.gz
-          curl -L https://github.com/thomasbtf/small-kraken-db/raw/master/ont_reads.fastq.gz > .tests/data/ont_reads.fastq.gz
           curl -L https://github.com/thomasbtf/small-kraken-db/raw/master/ERR5745913.fastq.gz > .tests/data/ion_reads.fastq.gz
           echo sample_name,fq1,fq2,date,is_amplicon_data,technology > .tests/config/pep/samples.csv
           echo illumina-test,data/B117.1.fastq.gz,data/B117.2.fastq.gz,2022-01-01,$AMPLICON,illumina >> .tests/config/pep/samples.csv
-          echo ont-test,data/ont_reads.fastq.gz,,2022-01-01,$AMPLICON,ont >> .tests/config/pep/samples.csv
           echo ion-test,data/ion_reads.fastq.gz,,2022-01-01,$AMPLICON,ion >> .tests/config/pep/samples.csv
       - name: Prepare test data for Illumina
         if: steps.test-data.outputs.cache-hit != true && (startsWith(matrix.rule, 'all') && matrix.technology == 'illumina' || matrix.rule == 'compare_assemblers')
@@ -131,15 +137,14 @@ jobs:
           curl "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=nuccore&id=MZ314997.1&rettype=fasta" | sed '$ d' > .tests/resources/genomes/B.1.1.7.fasta
           curl "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=nuccore&id=MZ314998.1&rettype=fasta" | sed '$ d' > .tests/resources/genomes/B.1.351.fasta
       - name: Test rule ${{ matrix.rule }} on ${{ matrix.technology }} ${{ matrix.seq_method }} data
-        uses: snakemake/snakemake-github-action@v1.24.0
+        uses: snakemake/snakemake-github-action@v1.25.1
         with:
-          stagein: mamba env create -f workflow/envs/trinity.yaml -n trinity
           directory: .tests
           snakefile: workflow/Snakefile
           args: "-pr --use-conda --show-failed-logs --cores 2 --resources ncbi_api_requests=1 --conda-cleanup-pkgs cache --conda-frontend mamba ${{ matrix.rule }}"
 
       - name: Test report
-        uses: snakemake/snakemake-github-action@v1.24.0
+        uses: snakemake/snakemake-github-action@v1.25.1
         if: startsWith(matrix.rule, 'all -npr') != true
         with:
           directory: .tests
@@ -173,7 +178,7 @@ jobs:
     needs:
       - Formatting
       - Linting
-      - Pre-Commit
+      #- Pre-Commit
     strategy:
       matrix:
         rule:
@@ -184,10 +189,10 @@ jobs:
             benchmark_non_sars_cov_2,
             benchmark_reads,
             compare_assemblers,
-            generate_test_cases,
           ]
+    # generate_test_cases,
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       # - name: Cache conda dependencies
       #   uses: actions/cache@v2
@@ -310,14 +315,14 @@ jobs:
           curl "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=nuccore&id=MZ314997.1&rettype=fasta" | sed '$ d' > .tests/resources/genomes/B.1.1.7.fasta
           curl "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=nuccore&id=MZ314998.1&rettype=fasta" | sed '$ d' > .tests/resources/genomes/B.1.351.fasta
       - name: Test rule ${{ matrix.rule }}
-        uses: snakemake/snakemake-github-action@v1.24.0
+        uses: snakemake/snakemake-github-action@v1.25.1
         with:
           directory: .tests
           snakefile: workflow/Snakefile
           args: "-pr --use-conda --show-failed-logs --cores 2 --resources ncbi_api_requests=1 --conda-cleanup-pkgs cache --conda-frontend mamba ${{ matrix.rule }}"
 
       - name: Test report
-        uses: snakemake/snakemake-github-action@v1.24.0
+        uses: snakemake/snakemake-github-action@v1.25.1
         if: startsWith(matrix.rule, 'all -npr') != true
         with:
           directory: .tests

diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml
@@ -68,14 +68,14 @@ jobs:
           remove-android: "true"
 
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Space
         run: |
           echo "Free space:"
           df -h
 
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Checkout release PR locally
         if: ${{ steps.pr_created.outputs.pr == 'true' && steps.release.outputs.release_created != 'true' }}
@@ -111,7 +111,7 @@ jobs:
 
       - name: Create Dockerfile
         if: ${{ steps.release.outputs.release_created == 'true' }}
-        uses: snakemake/snakemake-github-action@v1.23.0
+        uses: snakemake/snakemake-github-action@v1.25.1
         with:
           directory: .
           snakefile: workflow/Snakefile
@@ -124,23 +124,23 @@ jobs:
 
       - name: Set up QEMU
         if: ${{ steps.release.outputs.release_created == 'true' }}
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
         if: ${{ steps.release.outputs.release_created == 'true' }}
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to Quay Registry
         if: ${{ steps.release.outputs.release_created == 'true' }}
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: quay.io
           username: uncovar+robot
           password: ${{ secrets.QUAYIO_PASSWORD }}
 
       - name: Build and push
         if: ${{ steps.release.outputs.release_created == 'true' }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true

diff --git a/.github/workflows/update-pangolin.yaml b/.github/workflows/update-pangolin.yaml
@@ -9,7 +9,7 @@ jobs:
   Create-PR-for-new-Pangolin-release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install mamba
         uses: conda-incubator/setup-miniconda@v2
@@ -37,7 +37,7 @@ jobs:
       - name: Create Pull Request
         if: ${{ env.LATEST_PANGOLIN != env.CURRENT_PANGOLIN }}
         id: cpr
-        uses: peter-evans/create-pull-request@v4
+        uses: peter-evans/create-pull-request@v5
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           committer: GitHub <[email protected]>

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,40 @@
 # Changelog
 
+## [0.16.0](https://github.com/IKIM-Essen/uncovar/compare/v0.15.1...v0.16.0) (2023-11-16)
+
+
+### Features
+
+* Update README.md for dark/light theme ([#614](https://github.com/IKIM-Essen/uncovar/issues/614)) ([33ac5f7](https://github.com/IKIM-Essen/uncovar/commit/33ac5f7b4f7af8bbb500e9f531ccbdc45ac9427f))
+
+
+### Bug Fixes
+
+* changes in path requirements for snakedeploy ([#590](https://github.com/IKIM-Essen/uncovar/issues/590)) ([406d456](https://github.com/IKIM-Essen/uncovar/commit/406d456edd15715f7e14338b8195d2a09a807e49))
+* improved clonality event definition, usage of major subclone for assembly polishing; upgrade to varlociraptor 6.0 ([#585](https://github.com/IKIM-Essen/uncovar/issues/585)) ([aced95e](https://github.com/IKIM-Essen/uncovar/commit/aced95e798fd0483e99120550f49614e680f56dd))
+* QA genome generation ([#613](https://github.com/IKIM-Essen/uncovar/issues/613)) ([2bb74bf](https://github.com/IKIM-Essen/uncovar/commit/2bb74bfc26dee80af7d04089aafe636ea7c6d76a))
+* switch from jupyter notebook to script ([#598](https://github.com/IKIM-Essen/uncovar/issues/598)) ([d0f293d](https://github.com/IKIM-Essen/uncovar/commit/d0f293d2b8136fb0d8ef3753733ab1d30fb20d56))
+* update multiqc ([#607](https://github.com/IKIM-Essen/uncovar/issues/607)) ([0070d89](https://github.com/IKIM-Essen/uncovar/commit/0070d8927290dd01f418b876b60f3405158f1ab9))
+* use homopolymer-pair-HMM-mode of Varlociraptor in case of nanopore reads ([505a271](https://github.com/IKIM-Essen/uncovar/commit/505a2714ba0e5642312051f5e5154f7d454ff57f))
+* use homopolymer-pair-HMM-mode of Varlociraptor in case of nanopore reads ([#587](https://github.com/IKIM-Essen/uncovar/issues/587)) ([9bc814f](https://github.com/IKIM-Essen/uncovar/commit/9bc814f7cb9e9d9e53d557d463e782801070b46e))
+
+
+### Performance Improvements
+
+* update to varlociraptor 8.4 ([#596](https://github.com/IKIM-Essen/uncovar/issues/596)) ([34701af](https://github.com/IKIM-Essen/uncovar/commit/34701af485aa1ec199a418ec1487a97677f624ca))
+
+## [0.15.1](https://github.com/IKIM-Essen/uncovar/compare/v0.15.0...v0.15.1) (2022-11-30)
+
+
+### Bug Fixes
+
+* **ci:** update black to rev 22.3.0 ([#523](https://github.com/IKIM-Essen/uncovar/issues/523)) ([2273b7d](https://github.com/IKIM-Essen/uncovar/commit/2273b7d49d21dfe48812f7e3e4d33969b62e1d9e))
+* disable ont actions ([#555](https://github.com/IKIM-Essen/uncovar/issues/555)) ([56e46ef](https://github.com/IKIM-Essen/uncovar/commit/56e46ef7e1663b4de3447068df7cb8b9d1d7b67f))
+* Replace outdated df append ([#544](https://github.com/IKIM-Essen/uncovar/issues/544)) ([4a40003](https://github.com/IKIM-Essen/uncovar/commit/4a4000310cba4f3e8640172eadebcd35239b2e34))
+* update channel priorities and snakemake wrappers ([#562](https://github.com/IKIM-Essen/uncovar/issues/562)) ([a0b6fd0](https://github.com/IKIM-Essen/uncovar/commit/a0b6fd0557b7e6cb3aeddd22e6a3350560c18415))
+* use latest varlociraptor release and ignore all biases in case of known lineage variant calling ([#575](https://github.com/IKIM-Essen/uncovar/issues/575)) ([fa957fd](https://github.com/IKIM-Essen/uncovar/commit/fa957fd44594920c12df3821a23905055b1de9c9))
+* voc probability ([#541](https://github.com/IKIM-Essen/uncovar/issues/541)) ([db6eb17](https://github.com/IKIM-Essen/uncovar/commit/db6eb175aa7c25948677a9b64258afd377016d8c))
+
 ## [0.15.0](https://github.com/IKIM-Essen/uncovar/compare/v0.14.0...v0.15.0) (2022-03-24)
 
 

diff --git a/README.md b/README.md
@@ -1,6 +1,12 @@
 # UnCoVar
 
-![UnCoVar2](https://user-images.githubusercontent.com/77535027/133610563-d190e25c-504e-4953-92dd-f84a5b4a1191.png)
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://github.com/IKIM-Essen/uncovar/assets/77535027/8e17c6fc-ff7a-4c25-afc9-7888036d693e">
+  <source media="(prefers-color-scheme: light)" srcset="https://github.com/IKIM-Essen/uncovar/assets/77535027/c99f5a94-749b-422e-b319-1e3700d40a8e">
+  <img alt="UnCoVar Logo dark/light">
+</picture>
+
 
 ## SARS-CoV-2 Variant Calling and Lineage Assignment
 

diff --git a/config/config.yaml b/config/config.yaml
@@ -72,12 +72,12 @@ assembly:
     # assemblers used for shotgun sequencing with on Illumina technology
     shotgun: "megahit-std"
     # assemblers used for amplicon sequencing with on Illumina technology
-    amplicon: "metaspades"
+    amplicon: "megahit-std"
   oxford nanopore:
     # assemblers used for shotgun sequencing with on Oxford Nanopore technology
     shotgun: "megahit-std"
     # assemblers used for amplicon sequencing with on Oxford Nanopore technology
-    amplicon: "spades"
+    amplicon: "megahit-std"
     # Medaka models are named to indicate i) the pore type, ii) the sequencing device (MinION or PromethION), iii) the basecaller variant, and iv) the basecaller version
     # with the format: {pore}_{device}_{caller variant}_{caller version}
     # See https://github.com/nanoporetech/medaka#models for more information.
@@ -86,7 +86,7 @@ assembly:
     # assemblers used for shotgun sequencing with on Ion Torrent technology
     shotgun: "megahit-std"
     # assemblers used for amplicon sequencing with on Torrent technology
-    amplicon: "spades"
+    amplicon: "megahit-std"
   # minimum posterior probability for a clonal variant to be included in the generated pseudo-assembly
   min-variant-prob: 0.95
 

diff --git a/version.txt b/version.txt
@@ -1 +1 @@
-0.15.0
+0.16.0
diff --git a/workflow/Snakefile b/workflow/Snakefile
@@ -21,13 +21,7 @@ report: "report/workflow.rst"
 container: "docker://condaforge/mambaforge"
 
 
-containerized: "quay.io/uncovar/uncovar#0.15.0"
-
-
-if config["strain-calling"]["use-gisaid"]:
-
-    envvars:
-        "GISAID_API_TOKEN",
+containerized: "quay.io/uncovar/uncovar#0.16.0"
 
 
 include: "rules/common.smk"
@@ -47,17 +41,15 @@ include: "rules/variant_filtration.smk"
 include: "rules/variant_report.smk"
 include: "rules/generate_output.smk"
 include: "rules/benchmarking.smk"
+include: "rules/long_read.smk"
+include: "rules/lineage_variant_calling.smk"
 
 
 if config["data-handling"]["use-data-handling"]:
 
     include: "rules/preprocessing.smk"
 
 
-include: "rules/long_read.smk"
-include: "rules/lineage_variant_calling.smk"
-
-
 if config["data-handling"]["archive-data"]:
 
     rule save_latest_run:

diff --git a/workflow/envs/abyss.yaml b/workflow/envs/abyss.yaml
@@ -1,5 +1,6 @@
 channels:
-  - bioconda
   - conda-forge
+  - bioconda
+  - nodefaults
 dependencies:
   - abyss =2.3.1
diff --git a/workflow/envs/bamclipper.yaml b/workflow/envs/bamclipper.yaml
@@ -1,6 +1,7 @@
 channels:
-  - bioconda
   - conda-forge
+  - bioconda
+  - nodefaults
 dependencies:
   - bamclipper =1.0
   - samtools =1.14 # see https://github.com/merenlab/anvio/issues/1479