132 nf quilt 10

.github/workflows/mega-linter.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
         with:
-          token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.GITHUB_TOKEN }} # secrets.PAT || 
           fetch-depth: 0 # If you use VALIDATE_ALL_CODEBASE = true, you can remove this line to improve performances
 
       # MegaLinter

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## [0.6.0] 2023-10-03
+
+- Stop overwriting README.md (use README_NF_QUILT.md instead)
+  - improve formatting
+- Smarter quilt_summarize.json
+  - Top level (only): md, html, pdf, csv, tsv
+  - multiqc sub-folder HTML
+
 ## [0.4.5] 2023-08-23
 
 - fix metadata in README.md

README.md

@@ -93,8 +93,8 @@ From the command-line, do, e.g.:
 
 ```bash
 # export NXF_VER=23.04.3
-export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.4.5/nf-quilt-0.4.5-meta.json
-nextflow run main.nf -plugins nf-quilt@0.4.5
+export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.6.0/nf-quilt-0.6.0-meta.json
+nextflow run main.nf -plugins nf-quilt@0.6.0
 ```
 
 For Tower, you can use the "Pre-run script" to set the environment variables.

plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy

@@ -43,6 +43,9 @@ import groovy.json.JsonOutput
 @Slf4j
 @CompileStatic
 class QuiltProduct {
+    public final static String README_FILE = 'README_NF_QUILT.md'
+    public final static String SUMMARY_FILE = 'quilt_summarize.json'
+
 
     private final static String KEY_META = 'metadata'
     private final static String KEY_README = 'readme'
@@ -51,19 +54,33 @@ class QuiltProduct {
 
     /* groovylint-disable-next-line GStringExpressionWithinString */
     private final static String DEFAULT_README = '''
-# ${now}
-## ${msg}
+# ${pkg}
+
+## ${now}
+
+## Run Command
+
+```bash
+${cmd}
+```
+
+### Workflow
 
-## workflow
-### scriptFile: ${meta['workflow']?.get('scriptFile')}
-### sessionId: ${meta['workflow']?.get('sessionId')}
-- start: ${meta['time_start']}
-- complete: ${meta['time_complete']}
+- **workflow run name**: ```${meta['workflow']?.get('runName')}```
+- **scriptFile**: ```${meta['workflow']?.get('scriptFile')}```
+- **sessionI**: ```${meta['workflow']?.get('sessionId')}```
+- **start**: ```${meta['time_start']}```
+- **complete**: ```${meta['time_complete']}```
 
-## processes
-${meta['workflow']['stats']['processes']}
+### Nextflow
+
+${nextflow}
+
+### Processes
+
+`${meta['workflow']['stats']['processes']}`
 '''
-    private final static String DEFAULT_SUMMARIZE = '**.md,**.html'
+    private final static String DEFAULT_SUMMARIZE = '*.md,*.html,*.?sv,*.pdf,multiqc/multiqc_report.html'
 
     private final static String[] BIG_KEYS = [
         'nextflow', 'commandLine', 'scriptFile', 'projectDir',
@@ -176,7 +193,7 @@ ${meta['workflow']['stats']['processes']}
             writeNextflowMetadata(params, 'params')
             params.remove('genomes')
             params.remove('test_data')
-            //printMap(params, 'params')
+            // printMap(params, 'params')
         }
         Map wf = session.getWorkflowMetadata().toMap()
         String start = wf['start']
@@ -190,7 +207,7 @@ ${meta['workflow']['stats']['processes']}
             wf.remove('complete')
             wf.remove('workflowStats')
             wf.remove('commandLine')
-            //printMap(wf, 'workflow')
+            // printMap(wf, 'workflow')
             log.info("\npublishing: ${wf['runName']}")
         }
         return [
@@ -206,28 +223,41 @@ ${meta['workflow']['stats']['processes']}
     String setupReadme() {
         String text = 'Stub README'
         try {
-            text = readme()
+            text = makeReadme()
         }
         catch (Exception e) {
             log.error("setupReadme failed: ${e.getMessage()}", pkg.meta)
         }
         if (text != null && text.length() > 0) {
             //log.debug("setupReadme: ${text.length()} bytes")
-            writeString(text, pkg, 'README.md')
+            writeString(text, pkg, README_FILE)
         }
         return text
     }
 
-    String readme() {
+    String makeReadme() {
         if (shouldSkip(KEY_README)) {
             log.info("readme=SKIP for ${pkg}")
             return null
         }
         GStringTemplateEngine engine = new GStringTemplateEngine()
         String raw_readme = pkg.meta_overrides(KEY_README, DEFAULT_README)
-        //log.debug("readme: ${raw_readme}")
-        Writable template = engine.createTemplate(raw_readme).make([meta: meta, msg: msg, now: now()])
-        return template.toString()
+        String cmd = "${meta['cmd']}".replace(' -', ' \\\n  -')
+        String nf = meta['workflow']?['nextflow']
+        String nextflow = nf?.replace(', ', '```\n  - **')\
+            ?.replace('nextflow.NextflowMeta(', '  - **')\
+            ?.replace(')', '```')
+            ?.replace(':', '**: ```')
+        String template = engine.createTemplate(raw_readme).make([
+            cmd: cmd,
+            meta: meta,
+            msg: msg,
+            nextflow: nextflow,
+            now: now(),
+            pkg: pkg.packageName,
+        ]).toString()
+        log.debug("readme.template: ${template}")
+        return template
     }
 
     List<Path> match(String glob) throws IOException {
@@ -275,7 +305,7 @@ ${meta['workflow']['stats']['processes']}
         }
 
         String qs_json = JsonOutput.toJson(quilt_summarize.keySet() as String[])
-        writeString(qs_json, pkg, 'quilt_summarize.json')
+        writeString(qs_json, pkg, SUMMARY_FILE)
         return quilt_summarize
     }
 

plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF

@@ -1,7 +1,7 @@
 Manifest-Version: 1.0
 Plugin-Class: nextflow.quilt.QuiltPlugin
 Plugin-Id: nf-quilt
-Plugin-Version: 0.4.5
+Plugin-Version: 0.6.0
 Plugin-Provider: Quilt Data
 Plugin-Requires: >=22.10.6
 

plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy

@@ -20,9 +20,14 @@ import nextflow.Session
 import nextflow.quilt.QuiltSpecification
 import nextflow.quilt.QuiltProduct
 import nextflow.quilt.jep.QuiltParser
+import nextflow.quilt.jep.QuiltPackage
 
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
 import groovy.transform.CompileDynamic
 import spock.lang.IgnoreIf
+import spock.lang.Unroll
 
 /**
  *
@@ -145,4 +150,45 @@ class QuiltProductTest extends QuiltSpecification {
         makeWriteProduct(skip_meta).pubStatus == 1 // still fails on implicit metadata
     }
 
+    void writeFile(root, filename) {
+        Path outPath = Paths.get(root, filename)
+        outPath.getParent().toFile().mkdirs()
+        Files.writeString(outPath, "#Time, Filename\n${timestamp},${filename}")
+        println("writeFile: ${filename} -> ${outPath}")
+    }
+
+    int writeFiles(dest) {
+        String root  = dest.toString()
+        String[] filenames = [
+            'SUMMARIZE_ME.md',
+            'SUMMARIZE_ME.csv',
+            'SUMMARIZE_ME.tsv',
+            'SUMMARIZE_ME.html',
+            'SUMMARIZE_ME.pdf',
+            'SUMMARIZE_ME.xml',
+            'multiqc/multiqc_report.html'
+        ]
+        filenames.each { writeFile(root, it) }
+        return filenames.size()
+    }
+
+    @Unroll
+    @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET ==  null })
+    void 'should summarize top-level readable files + multiqc '() {
+        given:
+        String sumURL = writeableURL('summarized')
+        QuiltPackage sumPkg = writeablePackage('summarized')
+        writeFiles(sumPkg.packageDest())
+
+        and:
+        Session session = Mock(Session)
+        QuiltPath path = QuiltPathFactory.parse(sumURL)
+        QuiltProduct product = new QuiltProduct(path, session)
+
+        expect:
+        product.publish() == 0
+        sumPkg.install()
+        Files.exists(Paths.get(sumPkg.packageDest().toString(), QuiltProduct.SUMMARY_FILE))
+    }
+
 }

plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy

@@ -19,6 +19,7 @@ package nextflow.quilt.jep
 import nextflow.quilt.QuiltSpecification
 import nextflow.quilt.nio.QuiltPathFactory
 import nextflow.quilt.nio.QuiltPath
+import nextflow.quilt.QuiltProduct
 
 import spock.lang.Ignore
 import spock.lang.IgnoreIf
@@ -39,7 +40,7 @@ import groovy.transform.CompileDynamic
 class QuiltPackageTest extends QuiltSpecification {
 
     private final static String PACKAGE_URL = 'quilt+s3://quilt-example#package=examples%2fsmart-report@d68a7e9'
-    private final static String TEST_URL = PACKAGE_URL + '&path=README.md'
+    private final static String TEST_URL = PACKAGE_URL + "&path=README.md"
 
     private QuiltPathFactory factory
     private QuiltPath qpath
@@ -149,7 +150,7 @@ class QuiltPackageTest extends QuiltSpecification {
         given:
         def qout = factory.parseUri(TEST_URL)
         def opkg = qout.pkg()
-        def outPath = Paths.get(opkg.packageDest().toString(), 'README.md')
+        def outPath = Paths.get(opkg.packageDest().toString(), QuiltProduct.README_FILE)
         Files.writeString(outPath, "Time: ${timestamp}")
         expect:
         Files.exists(outPath)
@@ -171,7 +172,7 @@ class QuiltPackageTest extends QuiltSpecification {
     void 'should succeed pushing new files to writeable bucket '() {
         given:
         QuiltPackage opkg = writeablePackage('observer')
-        def outPath = Paths.get(opkg.packageDest().toString(), 'README.md')
+        def outPath = Paths.get(opkg.packageDest().toString(), QuiltProduct.README_FILE)
         Files.writeString(outPath, "Time: ${timestamp}")
         expect:
         Files.exists(outPath)