diff --git a/.github/workflows/build-test-release.yml b/.github/workflows/build-test-release.yml new file mode 100644 index 0000000..a146cf7 --- /dev/null +++ b/.github/workflows/build-test-release.yml @@ -0,0 +1,253 @@ +name: CI/CD + +on: + push: + branches: + - main + - '*\@[0-9]+.[0-9]+.[0-9]+*' # package development branch + +jobs: + build: + if: ${{ !contains(github.event.head_commit.message, '[no ci]') }} + runs-on: ubuntu-latest + outputs: + branch: ${{ steps.get_pkg_info.outputs.branch }} + pkg_name: ${{ steps.get_pkg_info.outputs.pkg_name }} + pkg_ver: ${{ steps.get_pkg_info.outputs.pkg_ver }} + docker_file: ${{ steps.get_pkg_info.outputs.docker_file }} + repo_lowercase: ${{ steps.repo_lowercase.outputs.lowercase }} + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.6 + uses: actions/setup-python@v2 + with: + python-version: 3.6 + + - name: Extract package name and version from branch name + id: get_pkg_info + shell: bash + run: | + echo "::set-output name=branch::$(echo ${GITHUB_REF#refs/heads/})" + PKG_NAME=$(echo ${GITHUB_REF#refs/heads/} | awk -F'@' '{print $1}') + PKG_VER=$(echo ${GITHUB_REF#refs/heads/} | awk -F'@' '{print $2}') + echo "::set-output name=pkg_name::$(echo $PKG_NAME)" + echo "::set-output name=pkg_ver::$(echo $PKG_VER)" + if [[ -f "./$PKG_NAME/Dockerfile" ]]; then + echo "::set-output name=docker_file::$(echo ./$PKG_NAME/Dockerfile)" + else + echo "::set-output name=docker_file::" + fi + + - name: Login to GitHub Container Registry + if: ${{ steps.get_pkg_info.outputs.branch != 'main' && steps.get_pkg_info.outputs.docker_file != ''}} + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.CR_PAT }} + + - id: repo_lowercase + uses: ASzc/change-string-case-action@v1 + with: + string: ${{ github.repository }} + + - name: Build image + if: ${{ steps.get_pkg_info.outputs.branch != 'main' && steps.get_pkg_info.outputs.docker_file != ''}} + uses: docker/build-push-action@v2 + with: + context: "./${{steps.get_pkg_info.outputs.pkg_name}}" + file: "${{steps.get_pkg_info.outputs.docker_file}}" + platforms: linux/amd64 + load: true + tags: | + ghcr.io/${{ steps.repo_lowercase.outputs.lowercase }}.${{ steps.get_pkg_info.outputs.pkg_name }}:${{ steps.get_pkg_info.outputs.pkg_ver }} + + - name: Push image to ghcr.io + id: push_to_ghcr + if: ${{ steps.get_pkg_info.outputs.branch != 'main' && steps.get_pkg_info.outputs.docker_file != ''}} + shell: bash + run: | + docker push ghcr.io/${{ steps.repo_lowercase.outputs.lowercase }}.${{ steps.get_pkg_info.outputs.pkg_name }}:${{ steps.get_pkg_info.outputs.pkg_ver }} | tee ./ghcr.io.stdout + IMAGE_SHA=$(cat ./ghcr.io.stdout | tr ' ' '\n' | grep 'sha256:' | awk -F':' '{print $2}') + echo "::set-output name=image_sha256::$(echo ${IMAGE_SHA})" + echo "::set-output name=created_at::$(date -u +%FT%TZ)" + + + test-and-release: + runs-on: ubuntu-latest + needs: [build] + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.6 + uses: actions/setup-python@v2 + with: + python-version: 3.6 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install wfpm + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements-test.txt ]; then pip install -r requirements-test.txt; fi + if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi + + - name: Install Nextflow + run: | + wget --tries=10 -qO- https://get.nextflow.io | bash + sudo chmod 755 nextflow + sudo mv nextflow /usr/local/bin/ + + - name: "Login to GitHub Container Registry" # normally shouldn't need to login, but new image when just created is private + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.CR_PAT }} + + - name: Run tests for all packages + if: ${{ needs.build.outputs.branch == 'main' }} + run: | + wfpm test + + - name: Run tests for the current package only + if: ${{ needs.build.outputs.branch != 'main' }} + run: | + cd ./${{needs.build.outputs.pkg_name}} + wfpm test + + - name: Check whether to proceed with release + id: to_release + run: | + if [[ \ + "${{ github.event.head_commit.message }}" = "Merge"[[:space:]]"pull"[[:space:]]"request"* && \ + "${{ github.event.head_commit.message }}" = *"@"* && \ + "${{ github.event.head_commit.message }}" = *"[release]"* && \ + "${{ needs.build.outputs.branch }}" = 'main' + ]]; then + echo "::set-output name=release::$(echo Y)" + else + echo "::set-output name=release::$(echo N)" + fi + + - name: Extract package name and version from commit message + if: ${{ steps.to_release.outputs.release == 'Y' }} + id: get_pkg_info + shell: bash + run: | + MERGED_BR=$(echo -e '${{ github.event.head_commit.message }}' | \ + { grep '^Merge pull request'||:; } | tr ' ' '\n' |{ grep '@'||:; } | awk -F'/' '{print $2}') + PKG_NAME=$(echo $MERGED_BR | awk -F'@' '{print $1}') + PKG_VER=$(echo $MERGED_BR | awk -F'@' '{print $2}') + echo "::set-output name=pkg_name::$(echo ${PKG_NAME})" + echo "::set-output name=pkg_ver::$(echo ${PKG_VER})" + if [[ -f "./$PKG_NAME/Dockerfile" ]]; then + echo "::set-output name=docker_file::$(echo ./$PKG_NAME/Dockerfile)" + echo "::set-output name=docker_image::$(echo ghcr.io/${{ needs.build.outputs.repo_lowercase }}.$PKG_NAME:$PKG_VER)" + else + echo "::set-output name=docker_file::" + echo "::set-output name=docker_image::" + fi + + - name: Prepare package release tarball + id: prep_assets + if: ${{ steps.to_release.outputs.release == 'Y' }} + shell: bash + run: | + ./scripts/cleanup_temp_files.sh # just in case + PKG_TAR=${{ steps.get_pkg_info.outputs.pkg_name }}.v${{ steps.get_pkg_info.outputs.pkg_ver }}.tar.gz + pushd ${{ steps.get_pkg_info.outputs.pkg_name }} + tar --exclude=wfpr_modules --dereference -czvf ../$PKG_TAR . + popd + echo "::set-output name=pkg_tar::$(echo ${PKG_TAR})" + echo "::set-output name=pkg_tar_sha::$(sha256sum ${PKG_TAR} |awk '{print $1}')" + echo "::set-output name=pkg_tar_size::$(ls -l ${PKG_TAR} |awk '{print $5}')" + echo "::set-output name=created_at::$(date -u +%FT%TZ)" + + - name: Create release + id: create_release + if: ${{ steps.to_release.outputs.release == 'Y' }} + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ steps.get_pkg_info.outputs.pkg_name }}.v${{ steps.get_pkg_info.outputs.pkg_ver }} + release_name: ${{ steps.get_pkg_info.outputs.pkg_name }}.v${{ steps.get_pkg_info.outputs.pkg_ver }} + body: | + * Release `${{ steps.get_pkg_info.outputs.pkg_name }}.v${{ steps.get_pkg_info.outputs.pkg_ver }}` (${{ github.sha }}) + * Package `${{ steps.prep_assets.outputs.pkg_tar }}` (sha256: `${{ steps.prep_assets.outputs.pkg_tar_sha }}`) + draft: false + prerelease: false + + - name: Prepare in pkg-release.json + if: ${{ steps.to_release.outputs.release == 'Y' }} + shell: bash + run: | + WFPM_VER=$(wfpm -v | cut -d ' ' -f 2) + if [[ '${{ steps.get_pkg_info.outputs.docker_file }}' != '' ]]; then + docker pull ${{ steps.get_pkg_info.outputs.docker_image }} | tee image.info + IMAGE_SHA=$(cat ./image.info | tr ' ' '\n' | grep 'sha256:' | awk -F':' '{print $2}') + else + IMAGE_SHA=$(echo) + fi + + # temporary solution + TAG="${{ steps.get_pkg_info.outputs.pkg_name }}.v${{ steps.get_pkg_info.outputs.pkg_ver }}" + ./scripts/prepare_package_release_json.py -p ${{ steps.get_pkg_info.outputs.pkg_name }}/pkg.json -d \ + " + { + \"_release\": { + \"created\": \"${{ steps.prep_assets.outputs.created_at }}\", + \"hash\": { + \"checksum_type\": \"sha1\", + \"checksume\": \"${{ github.sha }}\" + }, + \"tag\": \"$TAG\", + \"assets\": [ + { + \"filename\": \"$TAG.tar.gz\", + \"download_url\": \"https://github.com/${{ github.repository }}/releases/download/$TAG/$TAG.tar.gz\", + \"checksum\": \"${{ steps.prep_assets.outputs.pkg_tar_sha }}\", + \"checksum_type\": \"sha256\", + \"size\": ${{ steps.prep_assets.outputs.pkg_tar_size }} + }, + { + \"_NOTE_\": \"this file\", + \"filename\": \"pkg-release.json\", + \"download_url\": \"https://github.com/${{ github.repository }}/releases/download/$TAG/pkg-release.json\", + \"checksum\": null, + \"checksum_type\": null, + \"size\": null + } + ] + }, + \"_image_digest\": { + \"checksum\": \"$IMAGE_SHA\", + \"checksum_type\": \"sha256\" + }, + \"_wfpm_ver\": \"$WFPM_VER\" + } + " > pkg-release.json + + - name: Upload package release tarball + if: ${{ steps.to_release.outputs.release == 'Y' }} + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./${{ steps.prep_assets.outputs.pkg_tar }} + asset_name: ${{ steps.prep_assets.outputs.pkg_tar }} + asset_content_type: application/zip + + - name: Upload package release json + if: ${{ steps.to_release.outputs.release == 'Y' }} + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./pkg-release.json + asset_name: pkg-release.json + asset_content_type: application/json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..10260be --- /dev/null +++ b/.gitignore @@ -0,0 +1,70 @@ +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +.eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 +venv*/ +pyvenv*/ + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +.coverage.* +nosetests.xml +coverage.xml +htmlcov + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject +.idea +*.iml +*.komodoproject + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +docs/_build + +.DS_Store +*~ +.*.sw[po] +.build +.ve +.env +.cache +.pytest +.bootstrap +.appveyor.token +*.bak +*.log +.vscode +.python-version +.nextflow* +work +outdir +wfpr_modules/github.com/*/*/*/tests diff --git a/.wfpm b/.wfpm new file mode 100644 index 0000000..7091ec0 --- /dev/null +++ b/.wfpm @@ -0,0 +1,5 @@ +project_name: wftools-alignment-rna +license: MIT +repo_type: git +repo_server: github.com +repo_account: ratschlab diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..62842a5 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021, ratschlab + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/LICENSE-short b/LICENSE-short new file mode 100644 index 0000000..02fc84c --- /dev/null +++ b/LICENSE-short @@ -0,0 +1,19 @@ +Copyright (c) 2021, ratschlab + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..16d22f5 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# Workflow tool package RNA-Seq alignment + +Update this to describe your awesome project. diff --git a/nextflow.config b/nextflow.config new file mode 100644 index 0000000..a5bfac7 --- /dev/null +++ b/nextflow.config @@ -0,0 +1,10 @@ +manifest { + homePage = 'https://github.com/ratschlab/wftools-alignment-rna' + description = 'Workflow tool package RNA-Seq alignment' + nextflowVersion = '>=20.10' +} + +docker { + enabled = true + runOptions = '-u \$(id -u):\$(id -g)' +} diff --git a/scripts/cleanup_temp_files.sh b/scripts/cleanup_temp_files.sh new file mode 100755 index 0000000..cd8dc20 --- /dev/null +++ b/scripts/cleanup_temp_files.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +for f in `find . -type d |egrep '(/work$)|(/.nextflow$)|(/outdir$)|(.nextflow.log)'`; do + echo remove $f; + rm -fr $f; +done + +for f in `find . -type f |egrep '\.nextflow\.log'`; do + echo remove $f; + rm $f; +done + diff --git a/scripts/prepare_package_release_json.py b/scripts/prepare_package_release_json.py new file mode 100755 index 0000000..4ccde99 --- /dev/null +++ b/scripts/prepare_package_release_json.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +# this is a temporary solution + +import argparse +import json + + +def update_image_digest(package_meta, release_meta_str): + with open(package_meta, 'r') as f: + package_meta = json.load(f) + + release_meta = json.loads(release_meta_str) + + if 'container' in package_meta: + package_meta['container']['_image_digest'] = release_meta['_image_digest'] + package_meta['_release'] = release_meta['_release'] + package_meta['_wfpm_ver'] = release_meta.get('_wfpm_ver') + + return package_meta + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Update pkg.json to generate pkg-release.json ') + parser.add_argument('-p', dest='package_meta', type=str, required=True) + parser.add_argument('-d', dest='release_meta_str', type=str, required=True) + args = parser.parse_args() + + updated_meta = update_image_digest( + args.package_meta, + args.release_meta_str + ) + + print(json.dumps(updated_meta, indent=4, sort_keys=True)) diff --git a/test_data/README.md b/test_data/README.md new file mode 100644 index 0000000..529612f --- /dev/null +++ b/test_data/README.md @@ -0,0 +1,9 @@ +# Data for testing + +This folder keeps test data used by all packages of this project. You are free to choose +however way you'd like to organize the test data. However here is some general guidance: + +* keep test files small to save space and reduce test execution time +* specific files used by different packages should be symlinked to individual package test directory +* since one file can be used by multiple packages, be careful when update the files, it may work for +one package but break for others diff --git a/wfpr_modules/README.md b/wfpr_modules/README.md new file mode 100644 index 0000000..154aa06 --- /dev/null +++ b/wfpr_modules/README.md @@ -0,0 +1,10 @@ +# Workflow Package Installation Directory + +This directory contains all dependent workflow packages that are to be installed by the package +management tool: [WFPM](https://github.com/icgc-argo/wfpm) (WorkFlow Package Manager). Please +do NOT modify contents under this directory manually. + +Ideally the packages installed in this directory do not need to be checked into git repo, similar +to NPM's `node_modules` directory. Instead, all dependent packages/modules are to be installed +at deployment time. But let's keep the modules checked into git repo before deployment time +installation is supported.