From 6025188667eb16f824118d0691c2b0082f5b9e71 Mon Sep 17 00:00:00 2001 From: "Dr. Ernie Prabhakar" Date: Tue, 9 Jan 2024 14:46:54 -0800 Subject: [PATCH 1/6] Use local NF_BIN --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 888bf6a5..4b6ad401 100644 --- a/Makefile +++ b/Makefile @@ -63,11 +63,11 @@ test-all: clean compile-all check #coverage pkg-test: compile-all echo "$(TEST_URI)" - ./launch.sh run ./main.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)" + $(NF_BIN) run ./main.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)" pkg-fail: compile echo "$(TEST_URI)" - ./launch.sh run ./fail.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)" + $(NF_BIN) run ./fail.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)" tower-test: $(NF_BIN) $(NF_BIN) run "https://github.com/quiltdata/nf-quilt" -name local_einstein -with-tower -r main -latest --pub "$(TEST_URI)" From 18f57237b74710c1464a9635580f6079e9a6aa40 Mon Sep 17 00:00:00 2001 From: "Dr. Ernie Prabhakar" Date: Tue, 9 Jan 2024 19:40:30 -0800 Subject: [PATCH 2/6] fix linter errors --- .groovylintrc.json | 36 +++++++++++-------- .../main/nextflow/quilt/QuiltProduct.groovy | 8 ++--- .../nextflow/quilt/jep/QuiltPackage.groovy | 24 +++++++------ .../nextflow/quilt/QuiltProductTest.groovy | 2 +- .../quilt/jep/QuiltPackageTest.groovy | 4 +-- 5 files changed, 42 insertions(+), 32 deletions(-) diff --git a/.groovylintrc.json b/.groovylintrc.json index 7e971c84..efe62778 100644 --- a/.groovylintrc.json +++ b/.groovylintrc.json @@ -1,46 +1,49 @@ { "extends": "recommended", "rules": { - "MethodCount": { + "CatchException": { "enabled": false }, - "JavadocMissingParamDescription": { + "CatchThrowable": { "enabled": false }, - "JavadocMissingThrowsDescription": { + "ClassJavadoc": { "enabled": false }, - "JavadocEmptyReturnTag": { + "ClosureAsLastMethodParameter": { "enabled": false }, - "JavadocEmptyFirstLine": { + "DuplicateNumberLiteral": { "enabled": false }, - "JUnitPublicNonTestMethod": { + "DuplicateStringLiteral": { "enabled": false }, - "CatchException": { + "FieldTypeRequired": { "enabled": false }, - "CatchThrowable": { + "ImplicitClosureParameter": { "enabled": false }, - "ClassJavadoc": { + "JUnitPublicNonTestMethod": { "enabled": false }, - "ClosureAsLastMethodParameter": { + "JavaIoPackageAccess": { "enabled": false }, - "DuplicateNumberLiteral": { + "JavadocEmptyFirstLine": { "enabled": false }, - "DuplicateStringLiteral": { + "JavadocEmptyReturnTag": { "enabled": false }, - "FieldTypeRequired": { + "JavadocMissingParamDescription": { "enabled": false }, - "JavaIoPackageAccess": { + "JavadocMissingThrowsDescription": { + "enabled": false + }, + "MethodCount": { "enabled": false }, "MethodParameterTypeRequired": { @@ -61,6 +64,9 @@ "SpaceAroundMapEntryColon": { "enabled": false }, + "SpaceAroundOperator": { + "enabled": false + }, "SystemExit": { "enabled": false }, @@ -80,4 +86,4 @@ "enabled": false } } -} +} \ No newline at end of file diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index 6df35404..c0314654 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -43,10 +43,10 @@ import groovy.json.JsonOutput @Slf4j @CompileStatic class QuiltProduct { + public final static String README_FILE = 'README_NF_QUILT.md' public final static String SUMMARY_FILE = 'quilt_summarize.json' - private final static String KEY_META = 'metadata' private final static String KEY_README = 'readme' private final static String KEY_SKIP = 'SKIP' @@ -142,7 +142,7 @@ ${nextflow} pkg.push(msg, meta) } catch (Exception e) { - log.error("publish failed:", e) + log.error('publish failed:', e) print("FAILED: $pkg\n") return // throw e @@ -251,7 +251,7 @@ ${nextflow} nextflow: nextflow, now: now(), pkg: pkg.packageName, - ]).toString() + ]) log.debug("readme.template: ${template}") return template } @@ -296,7 +296,7 @@ ${nextflow} List paths = match(wildcard) paths.each { path -> String filename = path.getFileName() - Map entry = ["path": path.toString(), "title": filename] + Map entry = ['path': path.toString(), 'title': filename] quilt_summarize.add(entry) } } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index 24193fa5..f69e8cd4 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -1,3 +1,4 @@ +/* groovylint-disable ReturnNullFromCatchBlock */ /* * Copyright 2022, Quilt Data Inc * @@ -34,10 +35,9 @@ import com.quiltdata.quiltcore.Namespace import com.quiltdata.quiltcore.Manifest import com.quiltdata.quiltcore.key.LocalPhysicalKey import com.quiltdata.quiltcore.key.S3PhysicalKey -import com.quiltdata.quiltcore.workflows.WorkflowException -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ObjectNode @Slf4j @CompileStatic @@ -184,15 +184,19 @@ class QuiltPackage { S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) Registry registry = new Registry(registryPath) Namespace namespace = registry.getNamespace(packageName) - String resolvedHash = (hash == 'latest' || hash == null || hash == 'null') ? namespace.getHash('latest') : hash + String resolvedHash = (hash == 'latest' || hash == null || hash == 'null') + ? namespace.getHash('latest') + : hash log.info("hash: $hash -> $resolvedHash") Manifest manifest = namespace.getManifest(resolvedHash) manifest.install(dest) - log.info("done") + log.info('done') } catch (IOException e) { log.error("failed to install $packageName") - return null + // this is non-fatal error, so we don't want to stop the pipeline + /* groovylint-disable-next-line ReturnNullFromCatchBlock */ + return } installed = true @@ -229,7 +233,7 @@ class QuiltPackage { Manifest.Builder builder = Manifest.builder() Files.walk(packageDest()).filter(f -> Files.isRegularFile(f)).forEach(f -> { - System.out.println(f) + log.debug(f) String logicalKey = packageDest().relativize(f) LocalPhysicalKey physicalKey = new LocalPhysicalKey(f) long size = Files.size(f) @@ -237,14 +241,14 @@ class QuiltPackage { }); Map fullMeta = [ - "version": Manifest.VERSION, - "user_meta": meta + this.meta, + 'version': Manifest.VERSION, + 'user_meta': meta + this.meta, ] ObjectMapper mapper = new ObjectMapper() builder.setMetadata((ObjectNode)mapper.valueToTree(fullMeta)) Manifest m = builder.build() - log.debug("QuiltPackage.push", m) + log.debug('QuiltPackage.push', m) m.push(namespace, "nf-quilt:${today()}-${msg}", parsed.workflowName) } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy index d1a10909..829fac2c 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy @@ -183,7 +183,7 @@ class QuiltProductTest extends QuiltSpecification { } int writeFiles(dest) { - String root = dest.toString() + String root = dest String[] filenames = [ 'SUMMARIZE_ME.md', 'SUMMARIZE_ME.csv', diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy index 8fdd7aa0..685b2724 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy @@ -40,7 +40,7 @@ import groovy.transform.CompileDynamic class QuiltPackageTest extends QuiltSpecification { private final static String PACKAGE_URL = 'quilt+s3://quilt-example#package=examples%2fsmart-report@d68a7e9' - private final static String TEST_URL = PACKAGE_URL + "&path=README.md" + private final static String TEST_URL = PACKAGE_URL + '&path=README.md' private QuiltPathFactory factory private QuiltPath qpath @@ -158,7 +158,7 @@ class QuiltPackageTest extends QuiltSpecification { when: opkg.push() then: - thrown(java.io.IOException) + thrown(IOException) } @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) From 55634244015a07449a845e694b4e57be67077ef0 Mon Sep 17 00:00:00 2001 From: "Dr. Ernie Prabhakar" Date: Wed, 10 Jan 2024 14:41:13 -0800 Subject: [PATCH 3/6] print and rethrow --- plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index c0314654..f45bff70 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -142,10 +142,9 @@ ${nextflow} pkg.push(msg, meta) } catch (Exception e) { - log.error('publish failed:', e) + log.error("Exception: ${e}") print("FAILED: $pkg\n") - return - // throw e + throw new RuntimeException(e) } print("SUCCESS: $pkg\n") } From da64dd6337bb3d4a25c9aa3081bc3a1f6e0372f2 Mon Sep 17 00:00:00 2001 From: "Dr. Ernie Prabhakar" Date: Wed, 10 Jan 2024 14:42:31 -0800 Subject: [PATCH 4/6] Nextflow, not NextFlow --- CHANGELOG.md | 4 ++-- README.md | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3145767a..9e77d57c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,12 +87,12 @@ Beta release (not yet on nextflow-io/plugins) - Alpha: Call Benchling API after publishing package - accept "?commit_message=text+string&readme=GString" in Quilt+ URI -- Support NextFlow 23+ +- Support Nextflow 23+ ## [0.3.3] 2023-03-08 - Fail gracefully on `quilt3` errors -- Detailed NextFlow Tower instructions +- Detailed Nextflow Tower instructions - Call `quilt3` during integration tests - Add full Quilt+ URL support, including metadata in query string diff --git a/README.md b/README.md index 5230217c..0f27580a 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # nf-quilt -NextFlow plugin for reading and writing Quilt packages as a FileSystem +Nextflow plugin for reading and writing Quilt packages as a FileSystem -[`nf-quilt`](https://github.com/quiltdata/nf-quilt) (v0.3.2 or later) is a NextFlow [plugin](https://www.nextflow.io/docs/latest/plugins.html) +[`nf-quilt`](https://github.com/quiltdata/nf-quilt) (v0.3.2 or later) is a Nextflow [plugin](https://www.nextflow.io/docs/latest/plugins.html) developed by [Quilt Data](https://quiltdata.com/) that enables you read and write directly -to Quilt packages using `quilt+s3` URIs wherever your NextFlow pipeline currently use `s3` URIs. +to Quilt packages using `quilt+s3` URIs wherever your Nextflow pipeline currently use `s3` URIs. Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs. @@ -13,7 +13,7 @@ Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) p This plugin allows your existing pipelines, without modification, to read and write versioned Quilt packages stored on Amazon S3. -Use the following three steps to configure NextFlow Tower or your command-line environment. +Use the following three steps to configure Nextflow Tower or your command-line environment. [Note: versions 0.7.0 and later no longer require the `quilt3` Python client.] 1. Enable the `nf-quilt` plugin @@ -70,7 +70,7 @@ Note that `--key` on the command-line corresponds to `params.key` in your script 1. Optional: use a pre-release plugin -If a plugin is not yet available in the NextFlow plugin registry, you can use a pre-release version. +If a plugin is not yet available in the Nextflow plugin registry, you can use a pre-release version. From the command-line, do, e.g.: ```bash @@ -173,13 +173,13 @@ You can compile run unit tests with: make check ``` -### Verifying NextFlow +### Verifying Nextflow -If this is your first time using NextFlow, you may also need to install a recent +If this is your first time using Nextflow, you may also need to install a recent [version of Java](https://www.java.com/en/download/help/download_options.html) for your platform. -NextFlow itself will take care of all the other dependencies. +Nextflow itself will take care of all the other dependencies. -You can verify and compile NextFlow with: +You can verify and compile Nextflow with: ```bash make nextflow From fb15b372ed174bd0e9185c834038e552b74b486c Mon Sep 17 00:00:00 2001 From: "Dr. Ernie Prabhakar" Date: Wed, 10 Jan 2024 16:02:20 -0800 Subject: [PATCH 5/6] 0.7.6 --- CHANGELOG.md | 14 +++++++++----- README.md | 4 ++-- .../main/nextflow/quilt/jep/QuiltPackage.groovy | 2 +- .../nf-quilt/src/resources/META-INF/MANIFEST.MF | 2 +- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e77d57c..c48f397e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,19 +1,23 @@ # Changelog +## [0.7.6] 2024-01-10 UNOFFICIAL + +- Re-enable crash on failure + ## [0.7.5] 2023-10-17 - Grab all multiqc subfolders -## [0.7.4] 2023-10-16b +## [0.7.4] 2023-10-16b UNOFFICIAL - Actually print the exception stack trace - `make install` to enable nextflow testing -## [0.7.3] 2023-10-16a +## [0.7.3] 2023-10-16a UNOFFICIAL - Catch and log exception on push failure -## [0.7.2] 2023-10-16 +## [0.7.2] 2023-10-16 UNOFFICIAL - Throw exceptions on push failure @@ -26,7 +30,7 @@ - Officially QuiltCore 0.1.0 instead of Python -## [0.6.0] 2023-10-03 +## [0.6.0] 2023-10-03 UNOFFICIAL (interim release still using Python) @@ -36,7 +40,7 @@ - Top level (only): md, html, pdf, csv, tsv - multiqc sub-folder HTML -## [0.5.0] 2023-09-04 +## [0.5.0] 2023-09-04 UNOFFICIAL - Switch to quiltcore-java [NOTE: this pre-release does NOT check workflows] - Do not pre-install packages (only install before write) diff --git a/README.md b/README.md index 0f27580a..a2177483 100644 --- a/README.md +++ b/README.md @@ -75,8 +75,8 @@ From the command-line, do, e.g.: ```bash # export NXF_VER=23.04.3 -export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.7.5/nf-quilt-0.7.5-meta.json -nextflow run main.nf -plugins nf-quilt@0.7.5 +export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.7.6/nf-quilt-0.7.6-meta.json +nextflow run main.nf -plugins nf-quilt@0.7.6 ``` For Tower, you can use the "Pre-run script" to set the environment variables. diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index f69e8cd4..d54c0b80 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -233,7 +233,7 @@ class QuiltPackage { Manifest.Builder builder = Manifest.builder() Files.walk(packageDest()).filter(f -> Files.isRegularFile(f)).forEach(f -> { - log.debug(f) + log.debug("push: ${f} -> ${packageDest()}") String logicalKey = packageDest().relativize(f) LocalPhysicalKey physicalKey = new LocalPhysicalKey(f) long size = Files.size(f) diff --git a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF index 622c7d64..0a53cefd 100644 --- a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF @@ -1,7 +1,7 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.quilt.QuiltPlugin Plugin-Id: nf-quilt -Plugin-Version: 0.7.5 +Plugin-Version: 0.7.6 Plugin-Provider: Quilt Data Plugin-Requires: >=22.10.6 From 782849b62c12fc3ff221796eb9c6c05bb3f715f6 Mon Sep 17 00:00:00 2001 From: "Dr. Ernie Prabhakar" Date: Wed, 10 Jan 2024 16:04:55 -0800 Subject: [PATCH 6/6] e.printStackTrace() --- plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index f45bff70..43209e2a 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -144,6 +144,7 @@ ${nextflow} catch (Exception e) { log.error("Exception: ${e}") print("FAILED: $pkg\n") + e.printStackTrace() throw new RuntimeException(e) } print("SUCCESS: $pkg\n")