Merge pull request #82 from PMCC-BioinformaticsCore/v0.11.x

V0.11.x

.github/workflows/publish.yml

@@ -0,0 +1,33 @@
+name: Upload Python Package
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install .
+        pip install -e .[tests]
+        pip install -e .[ci]
+
+    - name: Test with nosetests
+      run: |
+        nosetests -s -w tests -a ci --ignore-files="local_*"
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: |
+        python setup.py sdist bdist_wheel
+        twine upload dist/*

.github/workflows/unit_tests.yml

@@ -0,0 +1,26 @@
+name: Unit Tests
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install .
+        pip install -e .[tests]
+    - name: Test with nosetests
+      run: |
+        nosetests -s -w tests -a ci --ignore-files="local_*"

.gitignore

@@ -10,3 +10,5 @@ janis_pipelines.bioinformatics.egg-info/
 pip-wheel-metadata/
 cromwell-executions/
 cromwell-workflow-logs/
+venv/
+.python-version

.pre-commit-config.yaml

@@ -3,7 +3,6 @@ repos:
     rev: stable
     hooks:
     - id: black
-      language_version: python3.7
 - repo: local
   hooks:
     - id: nosetests

janis_bioinformatics/__meta__.py

@@ -1,2 +1,2 @@
-__version__ = "v0.10.12"
+__version__ = "v0.11.0"
 description = "Bioinformatics tools for Janis; the Pipeline creation helper"

janis_bioinformatics/data_types/bam.py

@@ -1,3 +1,4 @@
+import subprocess
 from janis_core import File
 
 
@@ -12,6 +13,28 @@ def name():
     def doc(self):
         return "A binary version of a SAM file, http://software.broadinstitute.org/software/igv/bam"
 
+    @classmethod
+    def flagstat(cls, file_path: str):
+        command = ["samtools", "flagstat", file_path]
+        result = subprocess.run(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+        )
+
+        if result.stderr:
+            raise Exception(result.stderr)
+
+        return result.stdout
+
+    @classmethod
+    def equal(cls, file_path_1: str, file_path_2: str):
+        flagstat1 = cls.flagstat(file_path_1)
+        flagstat2 = cls.flagstat(file_path_2)
+
+        return flagstat1 == flagstat2
+
 
 class BamBai(Bam):
     @staticmethod

janis_bioinformatics/data_types/bed.py

@@ -1,4 +1,7 @@
 from janis_core import File
+from janis_unix.data_types import Gunzipped
+
+from janis_bioinformatics.data_types.tabix import FileTabix
 
 
 class Bed(File):
@@ -10,14 +13,16 @@ def name():
         return "bed"
 
 
-class BedTabix(File):
+class BedGz(Gunzipped):
     def __init__(self, optional=False):
-        super().__init__(optional=optional, extension=".bed.gz")
+        super().__init__(inner_type=Bed, optional=optional, extension=".bed.gz")
 
     @staticmethod
     def name():
-        return "BedTABIX"
+        return "BedGz"
+
 
+class BedTabix(FileTabix, BedGz):
     @staticmethod
-    def secondary_files():
-        return [".tbi"]
+    def name():
+        return "BedTABIX"

janis_bioinformatics/data_types/fasta.py

@@ -4,7 +4,9 @@
 
 class Fasta(File):
     def __init__(self, optional=False):
-        super().__init__(optional, extension=".fasta")
+        super().__init__(
+            optional, extension=".fasta", alternate_extensions={".fa", ".fna"}
+        )
 
     @staticmethod
     def name():

janis_bioinformatics/data_types/fastq.py

@@ -5,7 +5,9 @@
 
 class Fastq(File):
     def __init__(self, optional=False):
-        super().__init__(optional=optional, extension=".fastq")
+        super().__init__(
+            optional=optional, extension=".fastq", alternate_extensions={".fq"}
+        )
 
     @staticmethod
     def name():
@@ -20,7 +22,9 @@ def doc(self):
 
 class FastqGz(File):
     def __init__(self, optional=False):
-        super().__init__(optional=optional, extension=".fastq.gz")
+        super().__init__(
+            optional=optional, extension=".fastq.gz", alternate_extensions={".fq.gz"}
+        )
 
     @staticmethod
     def name():
@@ -69,7 +73,7 @@ def invalid_value_hint(self, meta):
 
 class FastqPairedEnd(Array):
     def __init__(self, optional=False):
-        super().__init__(File(optional=False, extension=".fastq"), optional=optional)
+        super().__init__(Fastq(optional=False), optional=optional)
 
     def id(self):
         if self.optional:

janis_bioinformatics/data_types/tabix.py

@@ -0,0 +1,7 @@
+from janis_unix.data_types.gunzipped import Gunzipped
+
+
+class FileTabix(Gunzipped):
+    @staticmethod
+    def secondary_files():
+        return [".tbi"]

janis_bioinformatics/data_types/vcf.py

@@ -1,6 +1,9 @@
 from abc import ABC
 
 from janis_core import File
+from janis_unix import Gunzipped
+
+from janis_bioinformatics.data_types.tabix import FileTabix
 
 
 class Vcf(File):
@@ -32,9 +35,9 @@ def secondary_files():
         return [".idx"]
 
 
-class CompressedVcf(File):
+class CompressedVcf(Gunzipped):
     def __init__(self, optional=False):
-        super().__init__(optional=optional, extension=".vcf.gz")
+        super().__init__(inner_type=Vcf, optional=optional, extension=".vcf.gz")
 
     @staticmethod
     def name():
@@ -44,15 +47,11 @@ def doc(self):
         return ".vcf.gz"
 
 
-class VcfTabix(CompressedVcf):
+class VcfTabix(CompressedVcf, FileTabix):
     @staticmethod
     def name():
         return "CompressedIndexedVCF"
 
-    @staticmethod
-    def secondary_files():
-        return [".tbi"]
-
     def doc(self):
         return ".vcf.gz with .vcf.gz.tbi file"
 

janis_bioinformatics/tools/babrahambioinformatics/fastqc/versions.py

@@ -63,7 +63,7 @@ class FastqcSingleScattered(BioinformaticsWorkflow):
     f"""
     FastQC doesn't return a Directory unless it's the single variant, but Janis will make
     you double scatter if you're processing an array of array of fastqs.
-    
+
     Note, this is bound to the LATEST version of FastQC: '{fastqc_single_instantiated.version()}'
     """
 

janis_bioinformatics/tools/bcftools/annotate/base.py

@@ -50,6 +50,8 @@
 
 class BcfToolsAnnotateBase(BcfToolsToolBase, ABC):
     def bind_metadata(self):
+        self.metadata.contributors = ["Michael Franklin"]
+        self.metadata.dateCreated = date(2019, 1, 24)
         self.metadata.dateUpdated = date(2019, 1, 24)
         self.metadata.doi = "http://www.ncbi.nlm.nih.gov/pubmed/19505943"
         self.metadata.citation = (

janis_bioinformatics/tools/bcftools/concat/base.py

@@ -48,6 +48,8 @@ def outputs(self):
     def bind_metadata(self):
         from datetime import date
 
+        self.metadata.contributors = ["Michael Franklin"]
+        self.metadata.dateCreated = date(2019, 9, 9)
         self.metadata.dateUpdated = date(2019, 9, 9)
         self.metadata.doi = "http://www.ncbi.nlm.nih.gov/pubmed/19505943"
         self.metadata.citation = (

janis_bioinformatics/tools/bcftools/index/base.py

@@ -85,7 +85,9 @@ def outputs(self):
     def bind_metadata(self):
         from datetime import date
 
+        self.metadata.contributors = ["Michael Franklin"]
         self.metadata.dateCreated = date(2019, 1, 24)
+        self.metadata.dateUpdated = date(2019, 1, 24)
         self.metadata.doi = "http://www.ncbi.nlm.nih.gov/pubmed/19505943"
         self.metadata.citation = (
             "Li H, Handsaker B, Wysoker A, Fennell T, Ruan J, Homer N, Marth G, Abecasis G, Durbin R, "

janis_bioinformatics/tools/bcftools/norm/base.py

@@ -46,6 +46,8 @@ def outputs(self):
     def bind_metadata(self):
         from datetime import date
 
+        self.metadata.contributors = ["Michael Franklin"]
+        self.metadata.dateCreated = date(2019, 1, 24)
         self.metadata.dateUpdated = date(2019, 1, 24)
         self.metadata.doi = "http://www.ncbi.nlm.nih.gov/pubmed/19505943"
         self.metadata.citation = (

janis_bioinformatics/tools/bcftools/sort/base.py

@@ -100,7 +100,7 @@ def outputs(self):
 
     def bind_metadata(self):
         return ToolMetadata(
-            contributors=None,
+            contributors=["Michael Franklin"],
             dateCreated=datetime(2019, 5, 9),
             dateUpdated=datetime(2019, 7, 11),
             institution=None,

janis_bioinformatics/tools/bcftools/view/base.py

@@ -50,6 +50,8 @@ class BcfToolsViewBase(BcfToolsToolBase, ABC):
     def bind_metadata(self):
         from datetime import date
 
+        self.metadata.contributors = ["Michael Franklin"]
+        self.metadata.dateCreated = date(2019, 1, 24)
         self.metadata.dateUpdated = date(2019, 1, 24)
         self.metadata.doi = "http://www.ncbi.nlm.nih.gov/pubmed/19505943"
         self.metadata.citation = (

janis_bioinformatics/tools/bedtools/coveragebed/base.py

@@ -25,12 +25,12 @@
 class BedToolsCoverageBedBase(BedToolsToolBase, ABC):
     def bind_metadata(self):
 
-        self.contributors = ["Jiaan Yu"]
+        self.metadata.contributors = ["Jiaan Yu"]
         self.metadata.dateUpdated = date(2020, 2, 26)
         self.metadata.dateCreated = date(2020, 2, 20)
         self.metadata.doi = None
         self.metadata.citation = None
-        self.keywords = ["bedtools", "coverageBed", "coverage"]
+        self.metadata.keywords = ["bedtools", "coverageBed", "coverage"]
         self.metadata.documentationUrl = (
             "https://bedtools.readthedocs.io/en/latest/content/tools/coverage.html"
         )

janis_bioinformatics/tools/bedtools/genomecoveragebed/base.py

@@ -24,12 +24,12 @@
 class BedToolsGenomeCoverageBedBase(BedToolsToolBase, ABC):
     def bind_metadata(self):
 
-        self.contributors = ["Jiaan Yu"]
+        self.metadata.contributors = ["Jiaan Yu"]
         self.metadata.dateUpdated = date(2020, 4, 1)
         self.metadata.dateCreated = date(2020, 4, 1)
         self.metadata.doi = None
         self.metadata.citation = None
-        self.keywords = ["bedtools", "genomecov", "genomeCoverageBed"]
+        self.metadata.keywords = ["bedtools", "genomecov", "genomeCoverageBed"]
         self.metadata.documentationUrl = (
             "https://bedtools.readthedocs.io/en/latest/content/tools/genomecov.html"
         )

janis_bioinformatics/tools/bedtools/intersectbed/base.py

@@ -23,12 +23,12 @@
 class BedToolsIntersectBedBase(BedToolsToolBase, ABC):
     def bind_metadata(self):
 
-        self.contributors = ["Jiaan Yu"]
+        self.metadata.contributors = ["Jiaan Yu"]
         self.metadata.dateUpdated = date(2020, 2, 26)
         self.metadata.dateCreated = date(2020, 2, 20)
         self.metadata.doi = None
         self.metadata.citation = None
-        self.keywords = ["bedtools", "intersect"]
+        self.metadata.keywords = ["bedtools", "intersect"]
         self.metadata.documentationUrl = (
             "https://bedtools.readthedocs.io/en/latest/content/tools/intersect.html"
         )

janis_bioinformatics/tools/bioinformaticstoolbase.py

@@ -1,3 +1,5 @@
+import os
+import sys
 from abc import ABC
 
 from janis_core import (

janis_bioinformatics/tools/bwa/index/base.py

@@ -39,15 +39,19 @@ def inputs(self):
                 "algorithm",
                 String(optional=True),
                 prefix="-a",
-                doc="BWT construction algorithm: bwtsw, is or rb2 [auto]",
+                doc="""\
+BWT construction algorithm: bwtsw, is or rb2 [auto]
+    - is	IS linear-time algorithm for constructing suffix array. It requires 5.37N memory where N is the size of the database. IS is moderately fast, but does not work with database larger than 2GB. IS is the default algorithm due to its simplicity. The current codes for IS algorithm are reimplemented by Yuta Mori.
+    - bwtsw	Algorithm implemented in BWT-SW. This method works with the whole human genome.
+""",
             ),
         ]
 
     def outputs(self):
         return [ToolOutput("out", FastaBwa, glob=InputSelector("reference"))]
 
     def memory(self, hints: Dict[str, Any]):
-        return 2
+        return 8
 
     def cpus(self, hints: Dict[str, Any]):
         return 1