Merge branch 'main' of https://github.com/quiltdata/nf-quilt

quiltdata · Jan 24, 2024 · 57e26a3 · 57e26a3
2 parents 6025188 + 5240fbb
commit 57e26a3
Show file tree

Hide file tree

Showing 11 changed files with 71 additions and 57 deletions.
diff --git a/.github/workflows/mega-linter.yml b/.github/workflows/mega-linter.yml
@@ -27,7 +27,7 @@ jobs:
     steps:
       # Git Checkout
       - name: Checkout Code
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
         with:
           token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}
           fetch-depth: 0 # If you use VALIDATE_ALL_CODEBASE = true, you can remove this line to improve performances

diff --git a/.github/workflows/pkg-test.yml b/.github/workflows/pkg-test.yml
@@ -27,7 +27,7 @@ jobs:
     steps:
     # Git Checkout
     - name: Checkout Code
-      uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
         fetch-depth: 0

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -27,7 +27,7 @@ jobs:
     steps:
     # Git Checkout
     - name: Checkout Code
-      uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
         fetch-depth: 0

diff --git a/.groovylintrc.json b/.groovylintrc.json
@@ -1,46 +1,49 @@
 {
     "extends": "recommended",
     "rules": {
-        "MethodCount": {
+        "CatchException": {
             "enabled": false
         },
-        "JavadocMissingParamDescription": {
+        "CatchThrowable": {
             "enabled": false
         },
-        "JavadocMissingThrowsDescription": {
+        "ClassJavadoc": {
             "enabled": false
         },
-        "JavadocEmptyReturnTag": {
+        "ClosureAsLastMethodParameter": {
             "enabled": false
         },
-        "JavadocEmptyFirstLine": {
+        "DuplicateNumberLiteral": {
             "enabled": false
         },
-        "JUnitPublicNonTestMethod": {
+        "DuplicateStringLiteral": {
             "enabled": false
         },
-        "CatchException": {
+        "FieldTypeRequired": {
             "enabled": false
         },
-        "CatchThrowable": {
+        "ImplicitClosureParameter": {
             "enabled": false
         },
-        "ClassJavadoc": {
+        "JUnitPublicNonTestMethod": {
             "enabled": false
         },
-        "ClosureAsLastMethodParameter": {
+        "JavaIoPackageAccess": {
             "enabled": false
         },
-        "DuplicateNumberLiteral": {
+        "JavadocEmptyFirstLine": {
             "enabled": false
         },
-        "DuplicateStringLiteral": {
+        "JavadocEmptyReturnTag": {
             "enabled": false
         },
-        "FieldTypeRequired": {
+        "JavadocMissingParamDescription": {
             "enabled": false
         },
-        "JavaIoPackageAccess": {
+        "JavadocMissingThrowsDescription": {
+            "enabled": false
+        },
+        "MethodCount": {
             "enabled": false
         },
         "MethodParameterTypeRequired": {
@@ -61,6 +64,9 @@
         "SpaceAroundMapEntryColon": {
             "enabled": false
         },
+        "SpaceAroundOperator": {
+            "enabled": false
+        },
         "SystemExit": {
             "enabled": false
         },
@@ -80,4 +86,4 @@
             "enabled": false
         }
     }
-}
+}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,19 +1,23 @@
 # Changelog
 
+## [0.7.6] 2024-01-10 UNOFFICIAL
+
+- Re-enable crash on failure
+
 ## [0.7.5] 2023-10-17
 
 - Grab all multiqc subfolders
 
-## [0.7.4] 2023-10-16b
+## [0.7.4] 2023-10-16b UNOFFICIAL
 
 - Actually print the exception stack trace
 - `make install` to enable nextflow testing
 
-## [0.7.3] 2023-10-16a
+## [0.7.3] 2023-10-16a UNOFFICIAL
 
 - Catch and log exception on push failure
 
-## [0.7.2] 2023-10-16
+## [0.7.2] 2023-10-16 UNOFFICIAL
 
 - Throw exceptions on push failure
 
@@ -26,7 +30,7 @@
 
 - Officially QuiltCore 0.1.0 instead of Python
 
-## [0.6.0] 2023-10-03
+## [0.6.0] 2023-10-03 UNOFFICIAL
 
 (interim release still using Python)
 
@@ -36,7 +40,7 @@
   - Top level (only): md, html, pdf, csv, tsv
   - multiqc sub-folder HTML
 
-## [0.5.0] 2023-09-04
+## [0.5.0] 2023-09-04 UNOFFICIAL
 
 - Switch to quiltcore-java [NOTE: this pre-release does NOT check workflows]
 - Do not pre-install packages (only install before write)
@@ -87,12 +91,12 @@ Beta release (not yet on nextflow-io/plugins)
 
 - Alpha: Call Benchling API after publishing package
 - accept "?commit_message=text+string&readme=GString" in Quilt+ URI
-- Support NextFlow 23+
+- Support Nextflow 23+
 
 ## [0.3.3] 2023-03-08
 
 - Fail gracefully on `quilt3` errors
-- Detailed NextFlow Tower instructions
+- Detailed Nextflow Tower instructions
 - Call `quilt3` during integration tests
 - Add full Quilt+ URL support, including metadata in query string
 

diff --git a/README.md b/README.md
@@ -1,10 +1,10 @@
 # nf-quilt
 
-NextFlow plugin for reading and writing Quilt packages as a FileSystem
+Nextflow plugin for reading and writing Quilt packages as a FileSystem
 
-[`nf-quilt`](https://github.com/quiltdata/nf-quilt) (v0.3.2 or later) is a NextFlow [plugin](https://www.nextflow.io/docs/latest/plugins.html)
+[`nf-quilt`](https://github.com/quiltdata/nf-quilt) (v0.3.2 or later) is a Nextflow [plugin](https://www.nextflow.io/docs/latest/plugins.html)
 developed by [Quilt Data](https://quiltdata.com/) that enables you read and write directly
-to Quilt packages using `quilt+s3` URIs wherever your NextFlow pipeline currently use `s3` URIs.
+to Quilt packages using `quilt+s3` URIs wherever your Nextflow pipeline currently use `s3` URIs.
 
 Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs.
 
@@ -13,7 +13,7 @@ Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) p
 This plugin allows your existing pipelines, without modification,
 to read and write versioned Quilt packages stored on Amazon S3.
 
-Use the following three steps to configure NextFlow Tower or your command-line environment.
+Use the following three steps to configure Nextflow Tower or your command-line environment.
 [Note: versions 0.7.0 and later no longer require the `quilt3` Python client.]
 
 1. Enable the `nf-quilt` plugin
@@ -70,13 +70,13 @@ Note that `--key` on the command-line corresponds to `params.key` in your script
 
 1. Optional: use a pre-release plugin
 
-If a plugin is not yet available in the NextFlow plugin registry, you can use a pre-release version.
+If a plugin is not yet available in the Nextflow plugin registry, you can use a pre-release version.
 From the command-line, do, e.g.:
 
 ```bash
 # export NXF_VER=23.04.3
-export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.7.5/nf-quilt-0.7.5-meta.json
-nextflow run main.nf -plugins [email protected].5
+export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.7.6/nf-quilt-0.7.6-meta.json
+nextflow run main.nf -plugins [email protected].6
 ```
 
 For Tower, you can use the "Pre-run script" to set the environment variables.
@@ -100,7 +100,7 @@ in order to customize the behavior of the plugin:
   * **metadata**: specify SKIP to not push any new metadata (implicit or explicit)
   * **_any other key_**: specify any other metadata key to store in the package
 
-See below for more details.  
+See below for more details.
 When running from the git repository,
 you can use the Makefile to test the various options,
 as long as you set a WRITE_BUCKET:
@@ -173,13 +173,13 @@ You can compile run unit tests with:
 make check
 ```
 
-### Verifying NextFlow
+### Verifying Nextflow
 
-If this is your first time using NextFlow, you may also need to install a recent
+If this is your first time using Nextflow, you may also need to install a recent
 [version of Java](https://www.java.com/en/download/help/download_options.html) for your platform.
-NextFlow itself will take care of all the other dependencies.
+Nextflow itself will take care of all the other dependencies.
 
-You can verify and compile NextFlow with:
+You can verify and compile Nextflow with:
 
 ```bash
 make nextflow

diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy
@@ -43,10 +43,10 @@ import groovy.json.JsonOutput
 @Slf4j
 @CompileStatic
 class QuiltProduct {
+
     public final static String README_FILE = 'README_NF_QUILT.md'
     public final static String SUMMARY_FILE = 'quilt_summarize.json'
 
-
     private final static String KEY_META = 'metadata'
     private final static String KEY_README = 'readme'
     private final static String KEY_SKIP = 'SKIP'
@@ -142,10 +142,10 @@ ${nextflow}
             pkg.push(msg, meta)
         }
         catch (Exception e) {
-            log.error("publish failed:", e)
+            log.error("Exception: ${e}")
             print("FAILED: $pkg\n")
-            return
-            // throw e
+            e.printStackTrace()
+            throw new RuntimeException(e)
         }
         print("SUCCESS: $pkg\n")
     }
@@ -251,7 +251,7 @@ ${nextflow}
             nextflow: nextflow,
             now: now(),
             pkg: pkg.packageName,
-        ]).toString()
+        ])
         log.debug("readme.template: ${template}")
         return template
     }
@@ -296,7 +296,7 @@ ${nextflow}
             List<Path> paths = match(wildcard)
             paths.each { path ->
                 String filename = path.getFileName()
-                Map entry = ["path": path.toString(), "title": filename]
+                Map entry = ['path': path.toString(), 'title': filename]
                 quilt_summarize.add(entry)
             }
         }

diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy
@@ -1,3 +1,4 @@
+/* groovylint-disable ReturnNullFromCatchBlock */
 /*
  * Copyright 2022, Quilt Data Inc
  *
@@ -34,10 +35,9 @@ import com.quiltdata.quiltcore.Namespace
 import com.quiltdata.quiltcore.Manifest
 import com.quiltdata.quiltcore.key.LocalPhysicalKey
 import com.quiltdata.quiltcore.key.S3PhysicalKey
-import com.quiltdata.quiltcore.workflows.WorkflowException
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.node.ObjectNode
 
 @Slf4j
 @CompileStatic
@@ -184,15 +184,19 @@ class QuiltPackage {
             S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null)
             Registry registry = new Registry(registryPath)
             Namespace namespace = registry.getNamespace(packageName)
-            String resolvedHash = (hash == 'latest' || hash == null || hash == 'null') ? namespace.getHash('latest') : hash
+            String resolvedHash = (hash == 'latest' || hash == null || hash == 'null')
+              ? namespace.getHash('latest')
+              : hash
             log.info("hash: $hash -> $resolvedHash")
             Manifest manifest = namespace.getManifest(resolvedHash)
 
             manifest.install(dest)
-            log.info("done")
+            log.info('done')
         } catch (IOException e) {
             log.error("failed to install $packageName")
-            return null
+            // this is non-fatal error, so we don't want to stop the pipeline
+            /* groovylint-disable-next-line ReturnNullFromCatchBlock */
+            return
         }
 
         installed = true
@@ -229,22 +233,22 @@ class QuiltPackage {
         Manifest.Builder builder = Manifest.builder()
 
         Files.walk(packageDest()).filter(f -> Files.isRegularFile(f)).forEach(f -> {
-            System.out.println(f)
+            log.debug("push: ${f} -> ${packageDest()}")
             String logicalKey = packageDest().relativize(f)
             LocalPhysicalKey physicalKey = new LocalPhysicalKey(f)
             long size = Files.size(f)
             builder.addEntry(logicalKey, new Entry(physicalKey, size, null, null))
         });
 
         Map<String, Object> fullMeta = [
-            "version": Manifest.VERSION,
-            "user_meta": meta + this.meta,
+            'version': Manifest.VERSION,
+            'user_meta': meta + this.meta,
         ]
         ObjectMapper mapper = new ObjectMapper()
         builder.setMetadata((ObjectNode)mapper.valueToTree(fullMeta))
 
         Manifest m = builder.build()
-        log.debug("QuiltPackage.push", m)
+        log.debug('QuiltPackage.push', m)
         m.push(namespace, "nf-quilt:${today()}-${msg}", parsed.workflowName)
     }
 

diff --git a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF
@@ -1,7 +1,7 @@
 Manifest-Version: 1.0
 Plugin-Class: nextflow.quilt.QuiltPlugin
 Plugin-Id: nf-quilt
-Plugin-Version: 0.7.5
+Plugin-Version: 0.7.6
 Plugin-Provider: Quilt Data
 Plugin-Requires: >=22.10.6
 
diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy
@@ -183,7 +183,7 @@ class QuiltProductTest extends QuiltSpecification {
     }
 
     int writeFiles(dest) {
-        String root  = dest.toString()
+        String root  = dest
         String[] filenames = [
             'SUMMARIZE_ME.md',
             'SUMMARIZE_ME.csv',

diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy
@@ -40,7 +40,7 @@ import groovy.transform.CompileDynamic
 class QuiltPackageTest extends QuiltSpecification {
 
     private final static String PACKAGE_URL = 'quilt+s3://quilt-example#package=examples%2fsmart-report@d68a7e9'
-    private final static String TEST_URL = PACKAGE_URL + "&path=README.md"
+    private final static String TEST_URL = PACKAGE_URL + '&path=README.md'
 
     private QuiltPathFactory factory
     private QuiltPath qpath
@@ -158,7 +158,7 @@ class QuiltPackageTest extends QuiltSpecification {
         when:
         opkg.push()
         then:
-        thrown(java.io.IOException)
+        thrown(IOException)
     }
 
     @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET ==  null })