From e3be5e1e99eea23552cdc137b3c4d6c2289aaac1 Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Wed, 19 Jul 2023 19:26:27 -0700 Subject: [PATCH 01/14] fix Makefile --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 134ac862..e599e704 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,6 @@ QUERY ?= ?Name=$(USER)&Owner=Kevin+Moore&Date=2023-03-07&Type=CRISPR&Notebook+UR TEST_URI ?= quilt+s3://$(WRITE_BUCKET)$(QUERY)\#package=test/hurdat$(FRAGMENT) QUILT_URI ?= quilt+s3://$(WRITE_BUCKET)\#package=$(PROJECT)/$(PIPELINE) PIP ?= pip3 -QUILT3 ?= /usr/local/bin/quilt3 REPORT ?= ./plugins/$(PROJECT)/build/reports/tests/test/index.html verify: #compile @@ -38,7 +37,7 @@ nextflow: pushd "$(NF_DIR)"; git checkout && make compile && git restore .; popd install-python: - if [ ! -x "$(QUILT3)" ]; then $(PIP) install quilt3 ; fi + if ! which quilt3; then $(PIP) install quilt3 ; fi which quilt3 compile-all: install-python nextflow compile From 38e2765e1258af91dfd33148313c51cc74261da9 Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Thu, 20 Jul 2023 02:07:51 -0700 Subject: [PATCH 02/14] WIP: Use quiltcore to install packages --- plugins/build.gradle | 1 + plugins/nf-quilt/build.gradle | 3 ++ .../nextflow/quilt/jep/QuiltPackage.groovy | 32 +++++++++++++++---- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/plugins/build.gradle b/plugins/build.gradle index 1e067cdd..ed48d2c5 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -108,6 +108,7 @@ subprojects { task copyPluginLibs(type: Sync) { from configurations.runtimeClasspath into 'build/target/libs' + duplicatesStrategy 'exclude' } /* diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index e7c747d0..3a20ea77 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -59,6 +59,9 @@ ext{ } dependencies { + // quiltcore + implementation zipTree('../../../quiltcore-java/lib/build/distributions/quiltcore.zip') + // This dependency is exported to consumers, that is to say found on their compile classpath. compileOnly "io.nextflow:nextflow:$nextflowVersion" compileOnly 'org.slf4j:slf4j-api:2.0.9' diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index 797d567a..45c92b89 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -28,6 +28,11 @@ import java.nio.file.attribute.BasicFileAttributes import java.util.stream.Collectors import java.time.LocalDate +import com.quiltdata.quiltcore.Registry +import com.quiltdata.quiltcore.Namespace +import com.quiltdata.quiltcore.Manifest +import com.quiltdata.quiltcore.key.S3PhysicalKey + @Slf4j @CompileStatic class QuiltPackage { @@ -86,7 +91,7 @@ class QuiltPackage { log.debug("${pkg}: attempting install for.pkgKey $pkgKey (okay if fails)") pkg.install() } - catch (Exception e) { + catch (IOException e) { log.warn("Package `${parsed.toUriString()}` does not yet exist") } return pkg @@ -250,17 +255,30 @@ class QuiltPackage { } } - // usage: quilt3 install [-h] [--registry REGISTRY] [--top-hash TOP_HASH] - // [--dest DEST] [--dest-registry DEST_REGISTRY] [--path PATH] name Path install() { - int exitCode = call('install', packageName, key_registry(), key_hash(), key_dest()) - if (exitCode != 0) { - log.warn("${exitCode}: ${packageName} failed to install (may not exist)") + Path dest = packageDest() + + try { + log.info("installing $packageName from $bucket...") + S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) + log.info("registryPath: $registryPath") + Registry registry = new Registry(registryPath) + Namespace namespace = registry.getNamespace(packageName) + String resolvedHash = (hash == 'latest' || hash == null || hash == 'null') ? namespace.getHash('latest') : hash + log.info("hash: $hash -> $resolvedHash") + Manifest manifest = namespace.getManifest(resolvedHash) + + manifest.install(dest) + log.info("done") + } catch (IOException e) { + log.error("failed to install $packageName") return null } + installed = true recursiveDeleteOnExit() - return packageDest() + + return dest } // https://stackoverflow.com/questions/15022219 From 4c5bcfb6a90e370ae63d4c04a9871aa4e2bda391 Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Tue, 22 Aug 2023 17:24:11 -0700 Subject: [PATCH 03/14] Replace push with quiltcore --- Makefile | 8 +- .../nextflow/quilt/jep/QuiltPackage.groovy | 112 ++++++------------ 2 files changed, 35 insertions(+), 85 deletions(-) diff --git a/Makefile b/Makefile index e599e704..35439606 100644 --- a/Makefile +++ b/Makefile @@ -4,12 +4,10 @@ WRITE_BUCKET ?= quilt-example FRAGMENT ?= &path=. NF_DIR ?= ../nextflow PID ?= $$$$ -PIP ?= python -m pip PIPELINE ?= sarek QUERY ?= ?Name=$(USER)&Owner=Kevin+Moore&Date=2023-03-07&Type=CRISPR&Notebook+URL=http%3A%2F%2Fexample.com TEST_URI ?= quilt+s3://$(WRITE_BUCKET)$(QUERY)\#package=test/hurdat$(FRAGMENT) QUILT_URI ?= quilt+s3://$(WRITE_BUCKET)\#package=$(PROJECT)/$(PIPELINE) -PIP ?= pip3 REPORT ?= ./plugins/$(PROJECT)/build/reports/tests/test/index.html verify: #compile @@ -36,11 +34,7 @@ nextflow: if [ ! -d "$(NF_DIR)" ]; then git clone https://github.com/nextflow-io/nextflow.git "$(NF_DIR)"; fi pushd "$(NF_DIR)"; git checkout && make compile && git restore .; popd -install-python: - if ! which quilt3; then $(PIP) install quilt3 ; fi - which quilt3 - -compile-all: install-python nextflow compile +compile-all: nextflow compile check: ./gradlew check --warning-mode all diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index 45c92b89..7ed0cc77 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -28,11 +28,16 @@ import java.nio.file.attribute.BasicFileAttributes import java.util.stream.Collectors import java.time.LocalDate +import com.quiltdata.quiltcore.Entry import com.quiltdata.quiltcore.Registry import com.quiltdata.quiltcore.Namespace import com.quiltdata.quiltcore.Manifest +import com.quiltdata.quiltcore.key.LocalPhysicalKey import com.quiltdata.quiltcore.key.S3PhysicalKey +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + @Slf4j @CompileStatic class QuiltPackage { @@ -48,7 +53,6 @@ class QuiltPackage { private final Path folder private final Map meta private boolean installed - private String cmd = 'N/A' static String today() { LocalDate date = LocalDate.now() @@ -171,51 +175,10 @@ class QuiltPackage { this.installed = false } - String key_dest() { - return "--dest ${packageDest()}" - } - - String key_dir() { - return "--dir ${packageDest()}" - } - boolean is_force() { return parsed.options[QuiltParser.P_FORCE] } - String key_force() { - //log.debug("key_force.options[${parsed.options[QuiltParser.P_FORCE]}] ${parsed.options}") - return is_force() ? '--force' : '' - } - - String key_hash() { - return (hash == 'latest' || hash == null || hash == 'null') ? '' : "--top-hash $hash" - } - - String key_meta(Map srcMeta = [:]) { - //log.debug("key_meta.srcMeta $srcMeta") - //log.debug("key_meta.uriMeta ${meta}") - Map metas = srcMeta + meta - if (metas.isEmpty()) { return '' } - - String jsonMeta = toJson(metas) - log.debug("key_meta.jsonMeta $jsonMeta") - return "--meta '$jsonMeta'" - } - - String key_msg(String message='') { - String msg = meta_overrides('msg', "nf-quilt:${today()}-${message}") - return "--message '${sanitize(msg)}'" - } - - String key_registry() { - return "--registry s3://${bucket}" - } - - String key_workflow() { - return (parsed.workflowName) ? "--workflow ${parsed.workflowName}" : '' - } - boolean isInstalled() { return installed } @@ -224,44 +187,12 @@ class QuiltPackage { return folder } - String lastCommand() { - return cmd - } - - int call(String... args) { - def command = ['quilt3'] - command.addAll(args) - cmd = command.join(' ') - log.debug("QuiltPackage.call `${cmd}`") - - try { - ProcessBuilder pb = new ProcessBuilder('bash', '-c', cmd) - pb.redirectErrorStream(true) - - Process p = pb.start() - //log.debug("call.start ${p}") - String result = new String(p.getInputStream().readAllBytes()) - //log.debug("call.result ${result}") - int exitCode = p.waitFor() - //log.debug("call.exitCode ${exitCode}") - if (exitCode != 0) { - log.debug("`call.fail` rc=${exitCode}[${cmd}]: ${result}\n") - } - return exitCode - } - catch (Exception e) { - log.warn("${e.getMessage()}: `${cmd}` ${this}") - return -1 - } - } - Path install() { Path dest = packageDest() try { log.info("installing $packageName from $bucket...") S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) - log.info("registryPath: $registryPath") Registry registry = new Registry(registryPath) Namespace namespace = registry.getNamespace(packageName) String resolvedHash = (hash == 'latest' || hash == null || hash == 'null') ? namespace.getHash('latest') : hash @@ -302,10 +233,35 @@ class QuiltPackage { } // https://docs.quiltdata.com/v/version-5.0.x/examples/gitlike#install-a-package int push(String msg = 'update', Map meta = [:]) { - int exitCode = 0 - String args = [key_dir(), key_registry(), key_meta(meta), key_msg(msg), key_force()].join(' ') - exitCode = call('push', packageName, args, key_workflow()) - return exitCode + S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) + Registry registry = new Registry(registryPath) + Namespace namespace = registry.getNamespace(packageName) + + Manifest.Builder builder = Manifest.builder() + + Files.walk(packageDest()).filter(f -> Files.isRegularFile(f)).forEach(f -> { + System.out.println(f) + String logicalKey = packageDest().relativize(f) + LocalPhysicalKey physicalKey = new LocalPhysicalKey(f) + long size = Files.size(f) + builder.addEntry(logicalKey, new Entry(physicalKey, size, null, null)) + }); + + Map fullMeta = [ + "version": Manifest.VERSION, + "user_meta": meta + this.meta, + ] + ObjectMapper mapper = new ObjectMapper() + builder.setMetadata((ObjectNode)mapper.valueToTree(fullMeta)) + + Manifest m = builder.build() + + try { + m.push(namespace, "nf-quilt:${today()}-${msg}") + } catch (IOException e) { + return 1 + } + return 0 } @Override From faad04dd3f061581a0db79e1a85c446e430f212b Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Tue, 22 Aug 2023 22:38:38 -0700 Subject: [PATCH 04/14] Use a maven repo at s3://quilt-dima --- plugins/nf-quilt/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index 3a20ea77..833a7833 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -38,6 +38,7 @@ repositories { maven { url = 'https://jitpack.io' } maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/releases' } maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/snapshots' } + maven { url = 'https://quilt-dima.s3.amazonaws.com/maven/releases' } } configurations { @@ -60,7 +61,7 @@ ext{ dependencies { // quiltcore - implementation zipTree('../../../quiltcore-java/lib/build/distributions/quiltcore.zip') + implementation 'com.quiltdata.quiltcore:quiltcore:0.0.1' // This dependency is exported to consumers, that is to say found on their compile classpath. compileOnly "io.nextflow:nextflow:$nextflowVersion" From f1e05709e1b109f5d28e6959955530cbddcba5a5 Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Mon, 4 Sep 2023 13:29:40 -0700 Subject: [PATCH 05/14] Bump quiltcore to 0.0.2 --- plugins/nf-quilt/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index 833a7833..c9d3ae5e 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -61,7 +61,7 @@ ext{ dependencies { // quiltcore - implementation 'com.quiltdata.quiltcore:quiltcore:0.0.1' + implementation 'com.quiltdata.quiltcore:quiltcore:0.0.2' // This dependency is exported to consumers, that is to say found on their compile classpath. compileOnly "io.nextflow:nextflow:$nextflowVersion" From 6575615b8851f5077c2a3299398c360a183ed280 Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Mon, 18 Sep 2023 07:16:20 -0700 Subject: [PATCH 06/14] Bump quiltcore and use workflows --- plugins/nf-quilt/build.gradle | 2 +- .../nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index c9d3ae5e..62ac06cd 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -61,7 +61,7 @@ ext{ dependencies { // quiltcore - implementation 'com.quiltdata.quiltcore:quiltcore:0.0.2' + implementation 'com.quiltdata.quiltcore:quiltcore:0.0.3' // This dependency is exported to consumers, that is to say found on their compile classpath. compileOnly "io.nextflow:nextflow:$nextflowVersion" diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index 7ed0cc77..5b36a8e8 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -257,7 +257,7 @@ class QuiltPackage { Manifest m = builder.build() try { - m.push(namespace, "nf-quilt:${today()}-${msg}") + m.push(namespace, "nf-quilt:${today()}-${msg}", parsed.workflowName) } catch (IOException e) { return 1 } From 2dd1d947eb022ee20ec30788ddb184cc59bc167c Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Mon, 18 Sep 2023 15:34:14 -0700 Subject: [PATCH 07/14] Bump quiltcore. Handle WorkflowException. --- plugins/nf-quilt/build.gradle | 2 +- .../nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index 62ac06cd..4c4bf6a4 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -61,7 +61,7 @@ ext{ dependencies { // quiltcore - implementation 'com.quiltdata.quiltcore:quiltcore:0.0.3' + implementation 'com.quiltdata.quiltcore:quiltcore:0.0.4' // This dependency is exported to consumers, that is to say found on their compile classpath. compileOnly "io.nextflow:nextflow:$nextflowVersion" diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index 5b36a8e8..57daef2a 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -34,6 +34,7 @@ import com.quiltdata.quiltcore.Namespace import com.quiltdata.quiltcore.Manifest import com.quiltdata.quiltcore.key.LocalPhysicalKey import com.quiltdata.quiltcore.key.S3PhysicalKey +import com.quiltdata.quiltcore.workflows.WorkflowException import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -258,6 +259,8 @@ class QuiltPackage { try { m.push(namespace, "nf-quilt:${today()}-${msg}", parsed.workflowName) + } catch (WorkflowException e) { + return 1 } catch (IOException e) { return 1 } From df4831bfd783e1ded8d7730f8d7d27687bcdb4fd Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Wed, 4 Oct 2023 19:37:15 -0700 Subject: [PATCH 08/14] Use quiltcore 0.1 from Maven Central --- plugins/nf-quilt/build.gradle | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index 4c4bf6a4..c40f1501 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -38,7 +38,6 @@ repositories { maven { url = 'https://jitpack.io' } maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/releases' } maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/snapshots' } - maven { url = 'https://quilt-dima.s3.amazonaws.com/maven/releases' } } configurations { @@ -61,7 +60,7 @@ ext{ dependencies { // quiltcore - implementation 'com.quiltdata.quiltcore:quiltcore:0.0.4' + implementation 'com.quiltdata:quiltcore:0.1' // This dependency is exported to consumers, that is to say found on their compile classpath. compileOnly "io.nextflow:nextflow:$nextflowVersion" From 67adaf345e9d62f9d9bb6315979a3a66b7c87477 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 4 Oct 2023 21:59:00 -0700 Subject: [PATCH 09/14] use realistic hash --- .../nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy index 0c6b0a92..9ace8dcc 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy @@ -60,7 +60,8 @@ class QuiltSpecification extends Specification { PluginExtensionProvider.reset() // this need to be set *before* the plugin manager class is created fullURL = 'quilt+s3://bkt?key=val&key2=val2' + - '#package=pre/suf@ab&path=p/t&property=prop&workflow=wf&catalog=quiltdata.com' + '#package=pre/suf@abcdef314159265'+ + '&path=p/t&property=prop&workflow=wf&catalog=quiltdata.com' pluginsMode = System.getProperty('pf4j.mode') timestamp = System.currentTimeMillis() writeBucket = System.getenv('WRITE_BUCKET') From d06e65da6627d9b19148f631a707803f9ad98072 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 4 Oct 2023 22:00:03 -0700 Subject: [PATCH 10/14] bump version --- CHANGELOG.md | 4 ++++ README.md | 4 ++-- plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 713cf674..b27b03f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [0.7.0] 2023-10-05 + +- Use QuiltCore 0.1.0 instead of Python + ## [0.6.0] 2023-10-03 - Stop overwriting README.md (use README_NF_QUILT.md instead) diff --git a/README.md b/README.md index 5892a478..b61f9f09 100644 --- a/README.md +++ b/README.md @@ -93,8 +93,8 @@ From the command-line, do, e.g.: ```bash # export NXF_VER=23.04.3 -export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.6.0/nf-quilt-0.6.0-meta.json -nextflow run main.nf -plugins nf-quilt@0.6.0 +export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.7.0/nf-quilt-0.7.0-meta.json +nextflow run main.nf -plugins nf-quilt@0.7.0 ``` For Tower, you can use the "Pre-run script" to set the environment variables. diff --git a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF index 85555939..ea14aa6c 100644 --- a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF @@ -1,7 +1,7 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.quilt.QuiltPlugin Plugin-Id: nf-quilt -Plugin-Version: 0.6.0 +Plugin-Version: 0.7.0 Plugin-Provider: Quilt Data Plugin-Requires: >=22.10.6 From 3cf2b02c613a29e5151db5298b221d0d1e66f47b Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 4 Oct 2023 22:02:04 -0700 Subject: [PATCH 11/14] remove Python from README --- README.md | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/README.md b/README.md index b61f9f09..eaf06eb6 100644 --- a/README.md +++ b/README.md @@ -15,25 +15,6 @@ to read and write versioned Quilt packages stored on Amazon S3. Use the following four steps to configure NextFlow Tower or your command-line environment. -1. Install the `quilt3` command-line tool - -This is distributed as an open source Python package you can install using `pip3`, -and must be available in the PATH used by `nextflow`. - -```bash -yum install python3-pip -y -yum install git -y -pip3 install quilt3 -which quilt3 -``` - -The above instructions use the 'yum' package manager, -which NextFlow Tower uses in the "Pre-run script" -when you edit the Pipeline settings from the Launchpad. - -If you are running from the command-line, you may need to use your own package manager -(or just skip those lines if you already have Python and Git). - 1. Enable the `nf-quilt` plugin The usual way to enable a plugin is to add the following to your `nextflow.config` file, @@ -183,13 +164,6 @@ git clone https://github.com/quiltdata/nf-quilt.git cd ./nf-quilt ``` -You also need to use Python to install the `quilt3` command-line tool used by `nf-quilt`: - -```bash -pip install quilt3 -which quilt3 -``` - ### Unit Testing You can compile run unit tests with: From e44fab0f7bbb07da1f11aa6491bd155f0d6a722b Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Thu, 5 Oct 2023 10:00:03 -0700 Subject: [PATCH 12/14] try prior CI tests --- .github/workflows/mega-linter.yml | 7 ++-- .github/workflows/pkg-test.yml | 61 +++++++++++++++++++++++++++++++ .github/workflows/test.yml | 16 ++++---- 3 files changed, 73 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/pkg-test.yml diff --git a/.github/workflows/mega-linter.yml b/.github/workflows/mega-linter.yml index 2a10ed4d..a58224a7 100644 --- a/.github/workflows/mega-linter.yml +++ b/.github/workflows/mega-linter.yml @@ -27,9 +27,9 @@ jobs: steps: # Git Checkout - name: Checkout Code - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 with: - token: ${{ secrets.GITHUB_TOKEN }} # secrets.PAT || + token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} fetch-depth: 0 # If you use VALIDATE_ALL_CODEBASE = true, you can remove this line to improve performances # MegaLinter @@ -37,7 +37,7 @@ jobs: id: ml # You can override MegaLinter flavor used to have faster performances # More info at https://megalinter.github.io/flavors/ - uses: oxsecurity/megalinter/flavors/dotnet@v7.4.0 + uses: oxsecurity/megalinter/flavors/dotnet@v7.3.0 env: # All available variables are described in documentation # https://megalinter.github.io/configuration/ @@ -48,6 +48,7 @@ jobs: # Upload MegaLinter artifacts - name: Archive production artifacts + if: ${{ success() }} || ${{ failure() }} uses: actions/upload-artifact@v3 with: name: MegaLinter reports diff --git a/.github/workflows/pkg-test.yml b/.github/workflows/pkg-test.yml new file mode 100644 index 00000000..51317a7c --- /dev/null +++ b/.github/workflows/pkg-test.yml @@ -0,0 +1,61 @@ +name: Test + +on: + # Trigger at every push. Action will also be visible from Pull Requests to master + push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions) + pull_request: + branches: [master] + +permissions: read-all + +jobs: + build: + name: Package Test + permissions: + contents: read + id-token: write + issues: write + pull-requests: write + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + java_version: [19] + runs-on: ${{ matrix.os }} + + steps: + # Git Checkout + - name: Checkout Code + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: arn:aws:iam::712023778557:role/GitHub-Testing-NF-Quilt + aws-region: us-east-1 + + - name: Setup Java ${{matrix.java_version}} + uses: actions/setup-java@v3 + with: + java-version: ${{matrix.java_version}} + distribution: 'temurin' + architecture: x64 + cache: gradle + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + + - name: Run Package Tests + run: make pkg-test + + - name: Archive production artifacts + if: ${{ success() }} || ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: nf-quilt-pkg-test + path: | + /home/runner/work/nf-quilt/nf-quilt/plugins/nf-quilt/build/reports/tests/test/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 809d703d..4f3a22e4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,8 +12,8 @@ jobs: build: name: Test permissions: - # contents: read - # id-token: write + contents: read + id-token: write issues: write pull-requests: write @@ -27,17 +27,16 @@ jobs: steps: # Git Checkout - name: Checkout Code - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 with: token: ${{ secrets.GITHUB_TOKEN }} fetch-depth: 0 - - name: Install Quilt3 - uses: actions/setup-python@v4 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v3 with: - python-version: '3.10' - - run: python -m pip install quilt3 - - run: which quilt3 + role-to-assume: arn:aws:iam::712023778557:role/GitHub-Testing-NF-Quilt + aws-region: us-east-1 - name: Setup Java ${{matrix.java_version}} uses: actions/setup-java@v3 @@ -54,6 +53,7 @@ jobs: run: make test - name: Archive production artifacts + if: ${{ success() }} || ${{ failure() }} uses: actions/upload-artifact@v3 with: name: nf-quilt-test-reports From 2525406c9ee65917ee070d5c2aac301dc178945b Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Thu, 5 Oct 2023 10:04:53 -0700 Subject: [PATCH 13/14] Fix README --- CHANGELOG.md | 9 ++++++++- README.md | 5 +++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b27b03f9..36d097b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,16 +2,23 @@ ## [0.7.0] 2023-10-05 -- Use QuiltCore 0.1.0 instead of Python +- Officially QuiltCore 0.1.0 instead of Python ## [0.6.0] 2023-10-03 +(interim release still using Python) + - Stop overwriting README.md (use README_NF_QUILT.md instead) - improve formatting - Smarter quilt_summarize.json - Top level (only): md, html, pdf, csv, tsv - multiqc sub-folder HTML +## [0.5.0] 2023-09-04 + +- Switch to quiltcore-java [NOTE: this pre-release does NOT check workflows] +- Do not pre-install packages (only install before write) + ## [0.4.5] 2023-08-23 - fix metadata in README.md diff --git a/README.md b/README.md index eaf06eb6..6c173b02 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,15 @@ NextFlow plugin for reading and writing Quilt packages as a FileSystem developed by [Quilt Data](https://quiltdata.com/) that enables you read and write directly to Quilt packages using `quilt+s3` URIs wherever your NextFlow pipeline currently use `s3` URIs. -Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs +Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs. ## I. Using the nf-quilt plugin in Production This plugin allows your existing pipelines, without modification, to read and write versioned Quilt packages stored on Amazon S3. -Use the following four steps to configure NextFlow Tower or your command-line environment. +Use the following three steps to configure NextFlow Tower or your command-line environment. +[Note: versions 0.7.0 and later no longer require the `quilt3` Python client.] 1. Enable the `nf-quilt` plugin From 109c7894bc32e220282e1f142b9eb63b2235ed4c Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Thu, 5 Oct 2023 10:05:56 -0700 Subject: [PATCH 14/14] try fixing pkg-test --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 35439606..471ac934 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ compile: nextflow: if [ ! -d "$(NF_DIR)" ]; then git clone https://github.com/nextflow-io/nextflow.git "$(NF_DIR)"; fi - pushd "$(NF_DIR)"; git checkout && make compile && git restore .; popd + cd "$(NF_DIR)"; git checkout && make compile && git restore .; cd .. compile-all: nextflow compile