From 6995e519486ee798a5082d8438d80e2028a8b7c6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 10 Jan 2024 14:41:38 -0800 Subject: [PATCH 1/3] Update actions/checkout digest to b4ffde6 (#156) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![Mend Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [actions/checkout](https://togithub.com/actions/checkout) | action | digest | `8ade135` -> `b4ffde6` | --- ### Configuration 📅 **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Mend Renovate](https://www.mend.io/free-developer-tools/renovate/). View repository job log [here](https://developer.mend.io/github/quiltdata/nf-quilt). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/mega-linter.yml | 2 +- .github/workflows/pkg-test.yml | 2 +- .github/workflows/test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mega-linter.yml b/.github/workflows/mega-linter.yml index 61d23dd6..88b4327e 100644 --- a/.github/workflows/mega-linter.yml +++ b/.github/workflows/mega-linter.yml @@ -27,7 +27,7 @@ jobs: steps: # Git Checkout - name: Checkout Code - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 with: token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} fetch-depth: 0 # If you use VALIDATE_ALL_CODEBASE = true, you can remove this line to improve performances diff --git a/.github/workflows/pkg-test.yml b/.github/workflows/pkg-test.yml index 51288e04..a9487157 100644 --- a/.github/workflows/pkg-test.yml +++ b/.github/workflows/pkg-test.yml @@ -27,7 +27,7 @@ jobs: steps: # Git Checkout - name: Checkout Code - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 with: token: ${{ secrets.GITHUB_TOKEN }} fetch-depth: 0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 86a29559..b241de5e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,7 @@ jobs: steps: # Git Checkout - name: Checkout Code - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 with: token: ${{ secrets.GITHUB_TOKEN }} fetch-depth: 0 From 7d118d13c8186385a03cf9a14e8e6eb31dc5b091 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 10 Jan 2024 23:41:55 +0100 Subject: [PATCH 2/3] Nextflow, not NextFlow (#160) Because I'm a pedant, sorry. --- CHANGELOG.md | 4 ++-- README.md | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3145767a..9e77d57c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,12 +87,12 @@ Beta release (not yet on nextflow-io/plugins) - Alpha: Call Benchling API after publishing package - accept "?commit_message=text+string&readme=GString" in Quilt+ URI -- Support NextFlow 23+ +- Support Nextflow 23+ ## [0.3.3] 2023-03-08 - Fail gracefully on `quilt3` errors -- Detailed NextFlow Tower instructions +- Detailed Nextflow Tower instructions - Call `quilt3` during integration tests - Add full Quilt+ URL support, including metadata in query string diff --git a/README.md b/README.md index 5230217c..21cde33a 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # nf-quilt -NextFlow plugin for reading and writing Quilt packages as a FileSystem +Nextflow plugin for reading and writing Quilt packages as a FileSystem -[`nf-quilt`](https://github.com/quiltdata/nf-quilt) (v0.3.2 or later) is a NextFlow [plugin](https://www.nextflow.io/docs/latest/plugins.html) +[`nf-quilt`](https://github.com/quiltdata/nf-quilt) (v0.3.2 or later) is a Nextflow [plugin](https://www.nextflow.io/docs/latest/plugins.html) developed by [Quilt Data](https://quiltdata.com/) that enables you read and write directly -to Quilt packages using `quilt+s3` URIs wherever your NextFlow pipeline currently use `s3` URIs. +to Quilt packages using `quilt+s3` URIs wherever your Nextflow pipeline currently use `s3` URIs. Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs. @@ -13,7 +13,7 @@ Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) p This plugin allows your existing pipelines, without modification, to read and write versioned Quilt packages stored on Amazon S3. -Use the following three steps to configure NextFlow Tower or your command-line environment. +Use the following three steps to configure Nextflow Tower or your command-line environment. [Note: versions 0.7.0 and later no longer require the `quilt3` Python client.] 1. Enable the `nf-quilt` plugin @@ -70,7 +70,7 @@ Note that `--key` on the command-line corresponds to `params.key` in your script 1. Optional: use a pre-release plugin -If a plugin is not yet available in the NextFlow plugin registry, you can use a pre-release version. +If a plugin is not yet available in the Nextflow plugin registry, you can use a pre-release version. From the command-line, do, e.g.: ```bash @@ -100,7 +100,7 @@ in order to customize the behavior of the plugin: * **metadata**: specify SKIP to not push any new metadata (implicit or explicit) * **_any other key_**: specify any other metadata key to store in the package -See below for more details. +See below for more details. When running from the git repository, you can use the Makefile to test the various options, as long as you set a WRITE_BUCKET: @@ -173,13 +173,13 @@ You can compile run unit tests with: make check ``` -### Verifying NextFlow +### Verifying Nextflow -If this is your first time using NextFlow, you may also need to install a recent +If this is your first time using Nextflow, you may also need to install a recent [version of Java](https://www.java.com/en/download/help/download_options.html) for your platform. -NextFlow itself will take care of all the other dependencies. +Nextflow itself will take care of all the other dependencies. -You can verify and compile NextFlow with: +You can verify and compile Nextflow with: ```bash make nextflow From 5240fbb26b0944a470ade8531685d7e2b20f2275 Mon Sep 17 00:00:00 2001 From: "Dr. Ernie Prabhakar" Date: Wed, 10 Jan 2024 16:53:06 -0800 Subject: [PATCH 3/3] chore: add back logging v0.7.6 (#171) The original error seems moot. `error unknown method invocation multiply on integer type` Still trying to get decent error messages on failure... --- .groovylintrc.json | 36 +++++++++++-------- CHANGELOG.md | 14 +++++--- Makefile | 4 +-- README.md | 4 +-- .../main/nextflow/quilt/QuiltProduct.groovy | 12 +++---- .../nextflow/quilt/jep/QuiltPackage.groovy | 24 +++++++------ .../src/resources/META-INF/MANIFEST.MF | 2 +- .../nextflow/quilt/QuiltProductTest.groovy | 2 +- .../quilt/jep/QuiltPackageTest.groovy | 4 +-- 9 files changed, 58 insertions(+), 44 deletions(-) diff --git a/.groovylintrc.json b/.groovylintrc.json index 7e971c84..efe62778 100644 --- a/.groovylintrc.json +++ b/.groovylintrc.json @@ -1,46 +1,49 @@ { "extends": "recommended", "rules": { - "MethodCount": { + "CatchException": { "enabled": false }, - "JavadocMissingParamDescription": { + "CatchThrowable": { "enabled": false }, - "JavadocMissingThrowsDescription": { + "ClassJavadoc": { "enabled": false }, - "JavadocEmptyReturnTag": { + "ClosureAsLastMethodParameter": { "enabled": false }, - "JavadocEmptyFirstLine": { + "DuplicateNumberLiteral": { "enabled": false }, - "JUnitPublicNonTestMethod": { + "DuplicateStringLiteral": { "enabled": false }, - "CatchException": { + "FieldTypeRequired": { "enabled": false }, - "CatchThrowable": { + "ImplicitClosureParameter": { "enabled": false }, - "ClassJavadoc": { + "JUnitPublicNonTestMethod": { "enabled": false }, - "ClosureAsLastMethodParameter": { + "JavaIoPackageAccess": { "enabled": false }, - "DuplicateNumberLiteral": { + "JavadocEmptyFirstLine": { "enabled": false }, - "DuplicateStringLiteral": { + "JavadocEmptyReturnTag": { "enabled": false }, - "FieldTypeRequired": { + "JavadocMissingParamDescription": { "enabled": false }, - "JavaIoPackageAccess": { + "JavadocMissingThrowsDescription": { + "enabled": false + }, + "MethodCount": { "enabled": false }, "MethodParameterTypeRequired": { @@ -61,6 +64,9 @@ "SpaceAroundMapEntryColon": { "enabled": false }, + "SpaceAroundOperator": { + "enabled": false + }, "SystemExit": { "enabled": false }, @@ -80,4 +86,4 @@ "enabled": false } } -} +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e77d57c..c48f397e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,19 +1,23 @@ # Changelog +## [0.7.6] 2024-01-10 UNOFFICIAL + +- Re-enable crash on failure + ## [0.7.5] 2023-10-17 - Grab all multiqc subfolders -## [0.7.4] 2023-10-16b +## [0.7.4] 2023-10-16b UNOFFICIAL - Actually print the exception stack trace - `make install` to enable nextflow testing -## [0.7.3] 2023-10-16a +## [0.7.3] 2023-10-16a UNOFFICIAL - Catch and log exception on push failure -## [0.7.2] 2023-10-16 +## [0.7.2] 2023-10-16 UNOFFICIAL - Throw exceptions on push failure @@ -26,7 +30,7 @@ - Officially QuiltCore 0.1.0 instead of Python -## [0.6.0] 2023-10-03 +## [0.6.0] 2023-10-03 UNOFFICIAL (interim release still using Python) @@ -36,7 +40,7 @@ - Top level (only): md, html, pdf, csv, tsv - multiqc sub-folder HTML -## [0.5.0] 2023-09-04 +## [0.5.0] 2023-09-04 UNOFFICIAL - Switch to quiltcore-java [NOTE: this pre-release does NOT check workflows] - Do not pre-install packages (only install before write) diff --git a/Makefile b/Makefile index 888bf6a5..4b6ad401 100644 --- a/Makefile +++ b/Makefile @@ -63,11 +63,11 @@ test-all: clean compile-all check #coverage pkg-test: compile-all echo "$(TEST_URI)" - ./launch.sh run ./main.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)" + $(NF_BIN) run ./main.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)" pkg-fail: compile echo "$(TEST_URI)" - ./launch.sh run ./fail.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)" + $(NF_BIN) run ./fail.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)" tower-test: $(NF_BIN) $(NF_BIN) run "https://github.com/quiltdata/nf-quilt" -name local_einstein -with-tower -r main -latest --pub "$(TEST_URI)" diff --git a/README.md b/README.md index 21cde33a..e1fec9a2 100644 --- a/README.md +++ b/README.md @@ -75,8 +75,8 @@ From the command-line, do, e.g.: ```bash # export NXF_VER=23.04.3 -export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.7.5/nf-quilt-0.7.5-meta.json -nextflow run main.nf -plugins nf-quilt@0.7.5 +export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.7.6/nf-quilt-0.7.6-meta.json +nextflow run main.nf -plugins nf-quilt@0.7.6 ``` For Tower, you can use the "Pre-run script" to set the environment variables. diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index 6df35404..43209e2a 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -43,10 +43,10 @@ import groovy.json.JsonOutput @Slf4j @CompileStatic class QuiltProduct { + public final static String README_FILE = 'README_NF_QUILT.md' public final static String SUMMARY_FILE = 'quilt_summarize.json' - private final static String KEY_META = 'metadata' private final static String KEY_README = 'readme' private final static String KEY_SKIP = 'SKIP' @@ -142,10 +142,10 @@ ${nextflow} pkg.push(msg, meta) } catch (Exception e) { - log.error("publish failed:", e) + log.error("Exception: ${e}") print("FAILED: $pkg\n") - return - // throw e + e.printStackTrace() + throw new RuntimeException(e) } print("SUCCESS: $pkg\n") } @@ -251,7 +251,7 @@ ${nextflow} nextflow: nextflow, now: now(), pkg: pkg.packageName, - ]).toString() + ]) log.debug("readme.template: ${template}") return template } @@ -296,7 +296,7 @@ ${nextflow} List paths = match(wildcard) paths.each { path -> String filename = path.getFileName() - Map entry = ["path": path.toString(), "title": filename] + Map entry = ['path': path.toString(), 'title': filename] quilt_summarize.add(entry) } } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index 24193fa5..d54c0b80 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -1,3 +1,4 @@ +/* groovylint-disable ReturnNullFromCatchBlock */ /* * Copyright 2022, Quilt Data Inc * @@ -34,10 +35,9 @@ import com.quiltdata.quiltcore.Namespace import com.quiltdata.quiltcore.Manifest import com.quiltdata.quiltcore.key.LocalPhysicalKey import com.quiltdata.quiltcore.key.S3PhysicalKey -import com.quiltdata.quiltcore.workflows.WorkflowException -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ObjectNode @Slf4j @CompileStatic @@ -184,15 +184,19 @@ class QuiltPackage { S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) Registry registry = new Registry(registryPath) Namespace namespace = registry.getNamespace(packageName) - String resolvedHash = (hash == 'latest' || hash == null || hash == 'null') ? namespace.getHash('latest') : hash + String resolvedHash = (hash == 'latest' || hash == null || hash == 'null') + ? namespace.getHash('latest') + : hash log.info("hash: $hash -> $resolvedHash") Manifest manifest = namespace.getManifest(resolvedHash) manifest.install(dest) - log.info("done") + log.info('done') } catch (IOException e) { log.error("failed to install $packageName") - return null + // this is non-fatal error, so we don't want to stop the pipeline + /* groovylint-disable-next-line ReturnNullFromCatchBlock */ + return } installed = true @@ -229,7 +233,7 @@ class QuiltPackage { Manifest.Builder builder = Manifest.builder() Files.walk(packageDest()).filter(f -> Files.isRegularFile(f)).forEach(f -> { - System.out.println(f) + log.debug("push: ${f} -> ${packageDest()}") String logicalKey = packageDest().relativize(f) LocalPhysicalKey physicalKey = new LocalPhysicalKey(f) long size = Files.size(f) @@ -237,14 +241,14 @@ class QuiltPackage { }); Map fullMeta = [ - "version": Manifest.VERSION, - "user_meta": meta + this.meta, + 'version': Manifest.VERSION, + 'user_meta': meta + this.meta, ] ObjectMapper mapper = new ObjectMapper() builder.setMetadata((ObjectNode)mapper.valueToTree(fullMeta)) Manifest m = builder.build() - log.debug("QuiltPackage.push", m) + log.debug('QuiltPackage.push', m) m.push(namespace, "nf-quilt:${today()}-${msg}", parsed.workflowName) } diff --git a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF index 622c7d64..0a53cefd 100644 --- a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF @@ -1,7 +1,7 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.quilt.QuiltPlugin Plugin-Id: nf-quilt -Plugin-Version: 0.7.5 +Plugin-Version: 0.7.6 Plugin-Provider: Quilt Data Plugin-Requires: >=22.10.6 diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy index d1a10909..829fac2c 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy @@ -183,7 +183,7 @@ class QuiltProductTest extends QuiltSpecification { } int writeFiles(dest) { - String root = dest.toString() + String root = dest String[] filenames = [ 'SUMMARIZE_ME.md', 'SUMMARIZE_ME.csv', diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy index 8fdd7aa0..685b2724 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy @@ -40,7 +40,7 @@ import groovy.transform.CompileDynamic class QuiltPackageTest extends QuiltSpecification { private final static String PACKAGE_URL = 'quilt+s3://quilt-example#package=examples%2fsmart-report@d68a7e9' - private final static String TEST_URL = PACKAGE_URL + "&path=README.md" + private final static String TEST_URL = PACKAGE_URL + '&path=README.md' private QuiltPathFactory factory private QuiltPath qpath @@ -158,7 +158,7 @@ class QuiltPackageTest extends QuiltSpecification { when: opkg.push() then: - thrown(java.io.IOException) + thrown(IOException) } @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null })