Quiltcore java demo

.github/workflows/mega-linter.yml

@@ -27,7 +27,7 @@ jobs:
     steps:
       # Git Checkout
       - name: Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
         with:
           token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}
           fetch-depth: 0 # If you use VALIDATE_ALL_CODEBASE = true, you can remove this line to improve performances

.github/workflows/pkg-test.yml

@@ -0,0 +1,61 @@
+name: Test
+
+on:
+  # Trigger at every push. Action will also be visible from Pull Requests to master
+  push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions)
+  pull_request:
+    branches: [master]
+
+permissions: read-all
+
+jobs:
+  build:
+    name: Package Test
+    permissions:
+      contents: read
+      id-token: write
+      issues: write
+      pull-requests: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        java_version: [19]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    # Git Checkout
+    - name: Checkout Code
+      uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        fetch-depth: 0
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v3
+      with:
+        role-to-assume: arn:aws:iam::712023778557:role/GitHub-Testing-NF-Quilt
+        aws-region: us-east-1
+
+    - name: Setup Java ${{matrix.java_version}}
+      uses: actions/setup-java@v3
+      with:
+        java-version: ${{matrix.java_version}}
+        distribution: 'temurin'
+        architecture: x64
+        cache: gradle
+
+    - name: Setup Gradle
+      uses: gradle/gradle-build-action@v2
+
+    - name: Run Package Tests
+      run: make pkg-test
+
+    - name: Archive production artifacts
+      if: ${{ success() }} || ${{ failure() }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: nf-quilt-pkg-test
+        path: |
+          /home/runner/work/nf-quilt/nf-quilt/plugins/nf-quilt/build/reports/tests/test/

.github/workflows/test.yml

@@ -12,8 +12,8 @@ jobs:
   build:
     name: Test
     permissions:
-      # contents: read
-      # id-token: write
+      contents: read
+      id-token: write
       issues: write
       pull-requests: write
 
@@ -27,17 +27,16 @@ jobs:
     steps:
     # Git Checkout
     - name: Checkout Code
-      uses: actions/checkout@v3
+      uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
         fetch-depth: 0
 
-    - name: Install Quilt3
-      uses: actions/setup-python@v4
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v3
       with:
-        python-version: '3.10' 
-    - run: python -m pip install quilt3
-    - run: which quilt3
+        role-to-assume: arn:aws:iam::712023778557:role/GitHub-Testing-NF-Quilt
+        aws-region: us-east-1
 
     - name: Setup Java ${{matrix.java_version}}
       uses: actions/setup-java@v3

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## [0.5.0] 2023-09-04+
+
+- Switch to quiltcore-java [NOTE: this pre-release does NOT check workflows]
+- Do not pre-install packages (only install before write)
+
 ## [0.4.5] 2023-08-23
 
 - fix metadata in README.md

Makefile

@@ -4,13 +4,10 @@ WRITE_BUCKET ?= quilt-example
 FRAGMENT ?= &path=.
 NF_DIR ?= ../nextflow
 PID ?= $$$$
-PIP ?= python -m pip
 PIPELINE ?= sarek
 QUERY ?= ?Name=$(USER)&Owner=Kevin+Moore&Date=2023-03-07&Type=CRISPR&Notebook+URL=http%3A%2F%2Fexample.com
 TEST_URI ?= quilt+s3://$(WRITE_BUCKET)$(QUERY)\#package=test/hurdat$(FRAGMENT)
 QUILT_URI ?=  quilt+s3://$(WRITE_BUCKET)\#package=$(PROJECT)/$(PIPELINE)
-PIP ?= pip3
-QUILT3 ?= /usr/local/bin/quilt3
 REPORT ?= ./plugins/$(PROJECT)/build/reports/tests/test/index.html
 
 verify: #compile
@@ -35,13 +32,9 @@ compile:
 
 nextflow:
 	if [ ! -d "$(NF_DIR)" ]; then git clone https://github.com/nextflow-io/nextflow.git  "$(NF_DIR)"; fi
-	pushd "$(NF_DIR)"; git checkout && make compile && git restore .; popd
+	cd "$(NF_DIR)" && git checkout && make compile && ./nextflow -v
 
-install-python:
-	if [ ! -x "$(QUILT3)" ]; then $(PIP) install quilt3 ; fi
-	which quilt3
-
-compile-all: install-python nextflow compile
+compile-all: nextflow compile
 
 check:
 	./gradlew check --warning-mode all
@@ -90,19 +83,17 @@ deps:
 
 deps-all:
 	./gradlew -q dependencyInsight
-#	./gradlew -q ${mm}dependencies --configuration ${CONFIG}  --dependency ${module}
 
 #
 # Refresh SNAPSHOTs dependencies
 #
+
 refresh:
 	./gradlew --refresh-dependencies dependencies
 
 #
 # Run all tests or selected ones
 #
-#test:
-#	./gradlew ${mm}test
 
 test-class:
 	./gradlew ${mm}test --tests ${class}
@@ -112,6 +103,7 @@ fast:
 
 #
 # Upload JAR artifacts to Maven Central
+#
 
 publish:
 	echo "Ensure you have set 'github_organization=<owner>' in gradle.properties"

README.md

@@ -6,33 +6,15 @@ NextFlow plugin for reading and writing Quilt packages as a FileSystem
 developed by [Quilt Data](https://quiltdata.com/) that enables you read and write directly
 to Quilt packages using `quilt+s3` URIs wherever your NextFlow pipeline currently use `s3` URIs.
 
-Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs
+Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs.
 
 ## I. Using the nf-quilt plugin in Production
 
 This plugin allows your existing pipelines, without modification,
 to read and write versioned Quilt packages stored on Amazon S3.
 
-Use the following four steps to configure NextFlow Tower or your command-line environment.
-
-1. Install the `quilt3` command-line tool
-
-This is distributed as an open source Python package you can install using `pip3`,
-and must be available in the PATH used by `nextflow`.
-
-```bash
-yum install python3-pip -y
-yum install git -y        
-pip3 install quilt3
-which quilt3
-```
-
-The above instructions use the 'yum' package manager,
-which NextFlow Tower uses in the "Pre-run script"
-when you edit the Pipeline settings from the Launchpad.
-
-If you are running from the command-line, you may need to use your own package manager
-(or just skip those lines if you already have Python and Git).
+Use the following three steps to configure NextFlow Tower or your command-line environment.
+[Note: versions 0.5.0 and later no longer require the `quilt3` Python client.]
 
 1. Enable the `nf-quilt` plugin
 
@@ -93,8 +75,8 @@ From the command-line, do, e.g.:
 
 ```bash
 # export NXF_VER=23.04.3
-export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.4.5/nf-quilt-0.4.5-meta.json
-nextflow run main.nf -plugins nf-quilt@0.4.5
+export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.5.0/nf-quilt-0.5.0-meta.json
+nextflow run main.nf -plugins nf-quilt@0.5.0
 ```
 
 For Tower, you can use the "Pre-run script" to set the environment variables.
@@ -183,13 +165,6 @@ git clone https://github.com/quiltdata/nf-quilt.git
 cd ./nf-quilt
 ```
 
-You also need to use Python to install the `quilt3` command-line tool used by `nf-quilt`:
-
-```bash
-pip install quilt3
-which quilt3
-```
-
 ### Unit Testing
 
 You can compile run unit tests with:

plugins/build.gradle

@@ -108,6 +108,7 @@ subprojects {
     task copyPluginLibs(type: Sync) {
         from configurations.runtimeClasspath
         into 'build/target/libs'
+        duplicatesStrategy 'exclude'
     }
 
     /*

plugins/nf-quilt/build.gradle

@@ -38,6 +38,7 @@ repositories {
     maven { url = 'https://jitpack.io' }
     maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/releases' }
     maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/snapshots' }
+    maven { url = 'https://quilt-dima.s3.amazonaws.com/maven/releases' }
 }
 
 configurations {
@@ -59,12 +60,15 @@ ext{
 }
 
 dependencies {
+    // quiltcore
+    implementation 'com.quiltdata.quiltcore:quiltcore:0.0.4'
+
     // This dependency is exported to consumers, that is to say found on their compile classpath.
     compileOnly "io.nextflow:nextflow:$nextflowVersion"
-    compileOnly 'org.slf4j:slf4j-api:2.0.7'
-    compileOnly 'org.pf4j:pf4j:3.9.0'
+    compileOnly 'org.slf4j:slf4j-api:2.0.9'
+    compileOnly 'org.pf4j:pf4j:3.10.0'
     // add here plugins depepencies
-    compileOnly 'org.slf4j:slf4j-simple:2.0.7'
+    compileOnly 'org.slf4j:slf4j-simple:2.0.9'
     compileOnly 'black.ninia:jep:4.1.1'
     runtime 'black.ninia:jep:4.1.1'
     runtimeOnly 'org.junit.platform:junit-platform-launcher:1.10.0'

plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy

@@ -78,7 +78,6 @@ class QuiltObserver implements TraceObserver {
         params.each { k, value ->
             String uri = "$value"
             if (uri.startsWith(QuiltParser.SCHEME)) {
-                log.debug("checkParams.uri[$k]: $uri")
                 QuiltPath path = QuiltPathFactory.parse(uri)
                 checkPath(path)
             }

plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy

@@ -80,11 +80,17 @@ ${meta['workflow']['stats']['processes']}
     static void writeString(String text, QuiltPackage pkg, String filename) {
         String dir = pkg.packageDest()
         Path path  = Paths.get(dir, filename.split('/') as String[])
+        File parent = path.getParent().toFile()
+        if (parent != null && !parent.exists() && !parent.mkdirs()) {
+            throw new IllegalStateException("Couldn't create dir: " + parent);
+        }
+
         try {
             Files.write(path, text.bytes)
         }
         catch (Exception e) {
-            log.error("writeString: cannot write `$text` to `$path` for `${pkg}`")
+            log.error("writeString[${e.getMessage()}]: fail write `$path` for `${pkg}`")
+            e.printStackTrace()
         }
     }
 
@@ -230,9 +236,9 @@ ${meta['workflow']['stats']['processes']}
         return template.toString()
     }
 
-    List<Path> match(String glob) throws IOException {
-        String dir = pkg.packageDest()
-        Path folder = Paths.get(dir)
+    List<Path> match(String glob, boolean shouldInstall = false) throws IOException {
+        log.debug("QuiltProduct.match[$shouldInstall]: $glob")
+        Path folder = shouldInstall ? pkg.install() : pkg.packageDest()
         FileSystem fs = FileSystems.getDefault()
         PathMatcher pathMatcher = fs.getPathMatcher("glob:${glob}")
         List<Path> matches = []