Merge changes from https://github.com/ohsu-comp-bio/funnel/tree/master

.github/dependabot.yml

@@ -4,5 +4,8 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
-    # Raise pull requests for version updates against the `develop` branch
     target-branch: "develop"
+    groups:
+      dependabot:
+        patterns:
+          - "*"

.github/workflows/compliance-test.yaml

@@ -0,0 +1,131 @@
+# Workflow for running the TES compliance suite against Funnel
+
+# This includes the following steps:
+#  1. Build Funnel and store the resulting binary artifact
+#  2. Install tes-compliance-suite and run against every version of TES simultaneously
+#  3. start-report-deployment: Send a dispatch to the funnel-compliance repository to generate and publish
+#     the tes-compliance-suite report to https://ohsu-comp-bio.github.io/funnel-compliance/
+
+# Optionally debug via SSH
+# Ref: https://fleetdm.com/engineering/tips-for-github-actions-usability
+#
+# To use this step uncomment and place anywhere in the build steps. The build will pause on this step and
+# output a ssh address associated with the Github action worker. Helpful for debugging build steps and
+# and intermediary files/artifacts.
+#
+# - name: "Debug: Package dependancies for tmate (CentOS)"
+#   run: |
+#     yum install -y xz
+#     ln -s /bin/true /bin/apt-get
+#
+# - name: Setup tmate session
+#   uses: mxschmitt/action-tmate@v3
+
+name: Compliance Test
+
+on: workflow_dispatch
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Build
+        run: make build
+      - name: Store Funnel
+        uses: actions/upload-artifact@v4
+        with:
+          name: funnelBin
+          path: funnel
+
+  compliance:
+    strategy:
+      fail-fast: false
+      matrix:
+        version: [1.0.0, 1.1.0]
+        db: ["boltdb", "mongodb"]
+        compute: ["local", "slurm"]
+    needs: build
+    runs-on: ubuntu-latest
+    container:
+      image: quay.io/ohsu-comp-bio/slurm
+      options: --hostname slurmctl --cap-add sys_admin
+    steps:
+      # Required to access the 'tests/mongo.config.yml' file
+      # Perhaps uploading it as an artifact would be more efficient?
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: funnelBin
+          path: .
+
+      - name: Start Funnel server
+        run: |
+          touch config.yml
+          if [ ${{ matrix.db }} = "mongodb" ]; then
+            make start-mongodb
+            cat `pwd`/tests/mongo.config.yml >> config.yml
+            # Required for Funnel to connect MongoDB
+            echo "172.17.0.1 localhost" >> /etc/hosts
+          elif [ ${{ matrix.compute }} = "slurm" ]; then
+            cat `pwd`/tests/slurm.config.yml >> config.yml
+            cp config.yml /opt/funnel_config.yml
+            # Start Slurm
+            /usr/local/bin/docker-entrypoint.sh
+          fi
+          chmod +x funnel
+          FLAGS="--config `pwd`/config.yml"
+          ./funnel server run $FLAGS &> funnel.logs &
+
+      - name: Run OpenAPI Test Runner
+        run: |
+          # Clone the 'upstream' OpenAPI Test Runner when PR #65 is merged
+          # https://github.com/elixir-cloud-aai/openapi-test-runner/pull/65
+          # git clone https://github.com/elixir-cloud-aai/openapi-test-runner
+          git clone https://github.com/ohsu-comp-bio/openapi-test-runner -b fix/create-and-filter-task
+
+          cd openapi-test-runner
+          python3 -m venv venv
+          source venv/bin/activate
+          pip install -r requirements.txt
+          python setup.py install
+          openapi-test-runner report --version "${{ matrix.version }}" --server "http://localhost:8000/"
+
+      - name: Install TES compliance suite
+        run: |
+          git clone https://github.com/lbeckman314/tes-compliance-suite -b feature/tesv1.1
+          cd tes-compliance-suite
+          /root/.pyenv/shims/python3 -m venv venv
+          source venv/bin/activate
+          pip install -r requirements.txt
+          python setup.py install
+          mkdir reports
+
+      - name: Test compliance (Report Publishing)
+        run: |
+          cd tes-compliance-suite
+          source venv/bin/activate
+          tes-compliance-suite report --version "${{ matrix.version }}" --server "http://localhost:8000/"
+
+  start-report-deployment:
+    needs: compliance
+    runs-on: ubuntu-latest
+    steps:
+      # https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#create-a-repository-dispatch-event
+      - name: Start report generation
+        uses: passeidireto/trigger-external-workflow-action@main
+        env:
+          PAYLOAD_AUTHOR: "Funnel"
+          PAYLOAD_REVISION: "3"
+        with:
+          repository: ohsu-comp-bio/funnel-compliance
+          event: start-report
+          github_pat: ${{ secrets.ACTIONS_TOKEN }}

.github/workflows/hugo.yml

@@ -0,0 +1,77 @@
+# Sample workflow for building and deploying a Hugo site to GitHub Pages
+name: Deploy Hugo site to Pages
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["master"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+# Default to bash
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    env:
+      HUGO_VERSION: 0.124.1
+    steps:
+      - name: Install Hugo CLI
+        run: |
+          wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb \
+          && sudo dpkg -i ${{ runner.temp }}/hugo.deb
+      - name: Install Dart Sass
+        run: sudo snap install dart-sass
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Setup Pages
+        id: pages
+        uses: actions/configure-pages@v5
+      - name: Install Node.js dependencies
+        run: "[[ -f package-lock.json || -f npm-shrinkwrap.json ]] && npm ci || true"
+      - name: Build with Hugo
+        env:
+          # For maximum backward compatibility with Hugo modules
+          HUGO_ENVIRONMENT: production
+          HUGO_ENV: production
+        run: |
+          hugo \
+            --minify \
+            --baseURL "${{ steps.pages.outputs.base_url }}/" \
+            --source website \
+            --destination public
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ./website/public
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.github/workflows/nextflow.yaml

@@ -0,0 +1,63 @@
+name: Nextflow Test
+
+on:
+  push:
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Build
+        run: make build
+      - name: Store Funnel
+        uses: actions/upload-artifact@v4
+        with:
+          name: funnelBin
+          path: funnel
+
+  nextflow:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+    - name: Download Funnel
+      uses: actions/download-artifact@v4
+      with:
+        name: funnelBin
+        path: .
+
+    - name: Start Funnel
+      run: |
+        chmod +x ./funnel
+        ./funnel server --LocalStorage.AllowedDirs $HOME run &
+
+    - name: Install Nextflow
+      run: |
+        cd ..
+        git clone https://github.com/nextflow-io/nextflow/ -b v24.04.4 --depth 1
+        cd nextflow
+        make compile
+
+    - name: Install nf-canary and GA4GH-TES plugin
+      run: |
+        cd ..
+        git clone https://github.com/seqeralabs/nf-canary
+        cd nf-canary
+        cat <<EOF >> nextflow.config
+          plugins {
+            id 'nf-ga4gh'
+          }
+          process.executor = 'tes'
+          tes.endpoint = 'http://localhost:8000'
+        EOF
+
+    - name: Run nf-canary tests
+      run: |
+        cd ../nf-canary
+        ../nextflow/launch.sh run main.nf

.github/workflows/s3-test.yaml

@@ -0,0 +1,48 @@
+# Credit: rhnvrm
+# Adapted from: https://rohanverma.net/blog/2021/02/09/minio-github-actions/
+
+name: S3 Integration Test
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  s3Test-integration:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup minio
+        run: |
+          docker run -d -p 9000:9000 --name minio \
+                     -e "MINIO_ROOT_USER=minioadmin" \
+                     -e "MINIO_ROOT_PASSWORD=minioadmin" \
+                     -v /tmp/data:/data \
+                     -v /tmp/config:/root/.minio \
+                     minio/minio server /data
+
+          cat <<EOF > funnel.config.yml
+          LocalStorage:
+            Disabled: true
+          AmazonS3:
+            Disabled: true
+          GoogleStorage:
+            Disabled: true
+          HTTPStorage:
+            Disabled: true
+          FTPStorage:
+            Disabled: true
+          GenericS3:
+            - Disabled: false
+              Endpoint: "localhost:9000"
+              Key: "minioadmin"
+              Secret: "minioadmin"
+          EOF
+
+          wget https://github.com/ohsu-comp-bio/funnel/releases/download/untagged-217841a99d14ccfe289c/funnel-darwin-arm64-0.11.0.tar.gz
+          tar -zxvf funnel-darwin-arm64-0.11.0.tar.gz
+          chmod +x funnel
+          ./funnel server run --config funnel.config.yml &
+          ./funnel task run examples/s3-test.yml