From 8e4156a6dff210ef6b2e0677eee94e752d756a0f Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Tue, 3 Oct 2023 10:12:20 -0700 Subject: [PATCH 1/8] plan 0.5.0 --- CHANGELOG.md | 8 ++++++++ README.md | 4 ++-- plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 929a2a7a..3603c293 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.5.0] 2023-10-03 + +- Stop overwriting README.md (use README_NF_QUILT.md instead) + - improve formatting +- Smarter quilt_summarize.json + - Top level (only): md, html, pdf, csv, tsv + - multiqc sub-folder HTML + ## [0.4.5] 2023-08-23 - fix metadata in README.md diff --git a/README.md b/README.md index 04cc61cf..84bd469e 100644 --- a/README.md +++ b/README.md @@ -93,8 +93,8 @@ From the command-line, do, e.g.: ```bash # export NXF_VER=23.04.3 -export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.4.5/nf-quilt-0.4.5-meta.json -nextflow run main.nf -plugins nf-quilt@0.4.5 +export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.5.0/nf-quilt-0.5.0-meta.json +nextflow run main.nf -plugins nf-quilt@0.5.0 ``` For Tower, you can use the "Pre-run script" to set the environment variables. diff --git a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF index ec3f6d05..2d73a071 100644 --- a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF @@ -1,7 +1,7 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.quilt.QuiltPlugin Plugin-Id: nf-quilt -Plugin-Version: 0.4.5 +Plugin-Version: 0.5.0 Plugin-Provider: Quilt Data Plugin-Requires: >=22.10.6 From e8616715997dcc531c103f09e871404196abc51b Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Tue, 3 Oct 2023 10:28:59 -0700 Subject: [PATCH 2/8] QuiltProduct.README_FILE --- .../nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy | 3 ++- .../src/test/nextflow/quilt/jep/QuiltPackageTest.groovy | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index add707f2..8efd9973 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -43,6 +43,7 @@ import groovy.json.JsonOutput @Slf4j @CompileStatic class QuiltProduct { + public final static String README_FILE = 'README.md' private final static String KEY_META = 'metadata' private final static String KEY_README = 'readme' @@ -213,7 +214,7 @@ ${meta['workflow']['stats']['processes']} } if (text != null && text.length() > 0) { //log.debug("setupReadme: ${text.length()} bytes") - writeString(text, pkg, 'README.md') + writeString(text, pkg, README_FILE) } return text } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy index 4e04570f..7a69dfea 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy @@ -19,6 +19,7 @@ package nextflow.quilt.jep import nextflow.quilt.QuiltSpecification import nextflow.quilt.nio.QuiltPathFactory import nextflow.quilt.nio.QuiltPath +import nextflow.quilt.QuiltProduct import spock.lang.Ignore import spock.lang.IgnoreIf @@ -39,7 +40,7 @@ import groovy.transform.CompileDynamic class QuiltPackageTest extends QuiltSpecification { private final static String PACKAGE_URL = 'quilt+s3://quilt-example#package=examples%2fsmart-report@d68a7e9' - private final static String TEST_URL = PACKAGE_URL + '&path=README.md' + private final static String TEST_URL = PACKAGE_URL + "&path=${QuiltProduct.README_FILE}" private QuiltPathFactory factory private QuiltPath qpath @@ -149,7 +150,7 @@ class QuiltPackageTest extends QuiltSpecification { given: def qout = factory.parseUri(TEST_URL) def opkg = qout.pkg() - def outPath = Paths.get(opkg.packageDest().toString(), 'README.md') + def outPath = Paths.get(opkg.packageDest().toString(), QuiltProduct.README_FILE) Files.writeString(outPath, "Time: ${timestamp}") expect: Files.exists(outPath) @@ -171,7 +172,7 @@ class QuiltPackageTest extends QuiltSpecification { void 'should succeed pushing new files to writeable bucket '() { given: QuiltPackage opkg = writeablePackage('observer') - def outPath = Paths.get(opkg.packageDest().toString(), 'README.md') + def outPath = Paths.get(opkg.packageDest().toString(), QuiltProduct.README_FILE) Files.writeString(outPath, "Time: ${timestamp}") expect: Files.exists(outPath) From f86ba23010074848bbb4c30c56960674904a2d92 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Tue, 3 Oct 2023 11:21:11 -0700 Subject: [PATCH 3/8] README_FILE = 'README_NF_QUILT.md' --- plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy | 2 +- .../src/test/nextflow/quilt/jep/QuiltPackageTest.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index 8efd9973..dbdb7c17 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -43,7 +43,7 @@ import groovy.json.JsonOutput @Slf4j @CompileStatic class QuiltProduct { - public final static String README_FILE = 'README.md' + public final static String README_FILE = 'README_NF_QUILT.md' private final static String KEY_META = 'metadata' private final static String KEY_README = 'readme' diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy index 7a69dfea..c3ca540a 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy @@ -40,7 +40,7 @@ import groovy.transform.CompileDynamic class QuiltPackageTest extends QuiltSpecification { private final static String PACKAGE_URL = 'quilt+s3://quilt-example#package=examples%2fsmart-report@d68a7e9' - private final static String TEST_URL = PACKAGE_URL + "&path=${QuiltProduct.README_FILE}" + private final static String TEST_URL = PACKAGE_URL + "&path=README.md" private QuiltPathFactory factory private QuiltPath qpath From 16f7f3c8e5bfc0a728f5a18936332be3b1381588 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Tue, 3 Oct 2023 15:48:45 -0700 Subject: [PATCH 4/8] new DEFAULT_SUMMARIZE '*.md,*.html,*.?sv,*.pdf,multiqc/multiqc_report.html' --- .../main/nextflow/quilt/QuiltProduct.groovy | 6 ++- .../nextflow/quilt/QuiltProductTest.groovy | 46 +++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index dbdb7c17..98e61441 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -44,6 +44,8 @@ import groovy.json.JsonOutput @CompileStatic class QuiltProduct { public final static String README_FILE = 'README_NF_QUILT.md' + public final static String SUMMARY_FILE = 'quilt_summarize.json' + private final static String KEY_META = 'metadata' private final static String KEY_README = 'readme' @@ -64,7 +66,7 @@ class QuiltProduct { ## processes ${meta['workflow']['stats']['processes']} ''' - private final static String DEFAULT_SUMMARIZE = '**.md,**.html' + private final static String DEFAULT_SUMMARIZE = '*.md,*.html,*.?sv,*.pdf,multiqc/multiqc_report.html' private final static String[] BIG_KEYS = [ 'nextflow', 'commandLine', 'scriptFile', 'projectDir', @@ -276,7 +278,7 @@ ${meta['workflow']['stats']['processes']} } String qs_json = JsonOutput.toJson(quilt_summarize.keySet() as String[]) - writeString(qs_json, pkg, 'quilt_summarize.json') + writeString(qs_json, pkg, SUMMARY_FILE) return quilt_summarize } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy index 398b6ca1..2eb0e0ab 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy @@ -20,9 +20,14 @@ import nextflow.Session import nextflow.quilt.QuiltSpecification import nextflow.quilt.QuiltProduct import nextflow.quilt.jep.QuiltParser +import nextflow.quilt.jep.QuiltPackage +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths import groovy.transform.CompileDynamic import spock.lang.IgnoreIf +import spock.lang.Unroll /** * @@ -145,4 +150,45 @@ class QuiltProductTest extends QuiltSpecification { makeWriteProduct(skip_meta).pubStatus == 1 // still fails on implicit metadata } + void writeFile(root, filename) { + Path outPath = Paths.get(root, filename) + outPath.getParent().toFile().mkdirs() + Files.writeString(outPath, "#Time, Filename\n${timestamp},${filename}") + println("writeFile: ${filename} -> ${outPath}") + } + + int writeFiles(dest) { + String root = dest.toString() + String[] filenames = [ + 'SUMMARIZE_ME.md', + 'SUMMARIZE_ME.csv', + 'SUMMARIZE_ME.tsv', + 'SUMMARIZE_ME.html', + 'SUMMARIZE_ME.pdf', + 'SUMMARIZE_ME.xml', + 'multiqc/multiqc_report.html' + ] + filenames.each { writeFile(root, it) } + return filenames.size() + } + + @Unroll + @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) + void 'should summarize top-level readable files + multiqc '() { + given: + String sumURL = writeableURL('summarized') + QuiltPackage sumPkg = writeablePackage('summarized') + writeFiles(sumPkg.packageDest()) + + and: + Session session = Mock(Session) + QuiltPath path = QuiltPathFactory.parse(sumURL) + QuiltProduct product = new QuiltProduct(path, session) + + expect: + product.publish() == 0 + sumPkg.install() + Files.exists(Paths.get(sumPkg.packageDest().toString(), QuiltProduct.SUMMARY_FILE)) + } + } From b3e757dd30e8f9367dfcb15ed479f273e59c1cb0 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Tue, 3 Oct 2023 15:50:31 -0700 Subject: [PATCH 5/8] v0.6.0 v0.5.0 used for quiltcore-java --- CHANGELOG.md | 2 +- README.md | 4 ++-- plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3603c293..713cf674 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [0.5.0] 2023-10-03 +## [0.6.0] 2023-10-03 - Stop overwriting README.md (use README_NF_QUILT.md instead) - improve formatting diff --git a/README.md b/README.md index 84bd469e..5892a478 100644 --- a/README.md +++ b/README.md @@ -93,8 +93,8 @@ From the command-line, do, e.g.: ```bash # export NXF_VER=23.04.3 -export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.5.0/nf-quilt-0.5.0-meta.json -nextflow run main.nf -plugins nf-quilt@0.5.0 +export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.6.0/nf-quilt-0.6.0-meta.json +nextflow run main.nf -plugins nf-quilt@0.6.0 ``` For Tower, you can use the "Pre-run script" to set the environment variables. diff --git a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF index 2d73a071..85555939 100644 --- a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF @@ -1,7 +1,7 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.quilt.QuiltPlugin Plugin-Id: nf-quilt -Plugin-Version: 0.5.0 +Plugin-Version: 0.6.0 Plugin-Provider: Quilt Data Plugin-Requires: >=22.10.6 From c74ed579e1bc64e3ec717b9b110bd6ea465a203d Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Tue, 3 Oct 2023 15:50:43 -0700 Subject: [PATCH 6/8] Update mega-linter.yml --- .github/workflows/mega-linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mega-linter.yml b/.github/workflows/mega-linter.yml index 61d23dd6..2a10ed4d 100644 --- a/.github/workflows/mega-linter.yml +++ b/.github/workflows/mega-linter.yml @@ -29,7 +29,7 @@ jobs: - name: Checkout Code uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4 with: - token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} # secrets.PAT || fetch-depth: 0 # If you use VALIDATE_ALL_CODEBASE = true, you can remove this line to improve performances # MegaLinter From 25206b3049149d8af0164d0a1117035d956b4c3a Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Tue, 3 Oct 2023 16:27:19 -0700 Subject: [PATCH 7/8] reformat README --- .../main/nextflow/quilt/QuiltProduct.groovy | 56 ++++++++++++++----- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index 98e61441..66bc3a78 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -54,17 +54,33 @@ class QuiltProduct { /* groovylint-disable-next-line GStringExpressionWithinString */ private final static String DEFAULT_README = ''' -# ${now} -## ${msg} +# ${pkg} -## workflow -### scriptFile: ${meta['workflow']?.get('scriptFile')} -### sessionId: ${meta['workflow']?.get('sessionId')} -- start: ${meta['time_start']} -- complete: ${meta['time_complete']} +## ${now} -## processes -${meta['workflow']['stats']['processes']} +## Run Command + +```bash +${cmd} +``` + +### Workflow + +- workflow run name: ```${meta['workflow']?.get('runName')}``` +- scriptFile: ```${meta['workflow']?.get('scriptFile')}``` +- sessionId: ```${meta['workflow']?.get('sessionId')}``` +- start: ```${meta['time_start']}``` +- complete: ```${meta['time_complete']}``` + +### Nextflow + +```bash +${nextflow} +``` + +### Processes + +`${meta['workflow']['stats']['processes']}` ''' private final static String DEFAULT_SUMMARIZE = '*.md,*.html,*.?sv,*.pdf,multiqc/multiqc_report.html' @@ -179,7 +195,7 @@ ${meta['workflow']['stats']['processes']} writeNextflowMetadata(params, 'params') params.remove('genomes') params.remove('test_data') - //printMap(params, 'params') + // printMap(params, 'params') } Map wf = session.getWorkflowMetadata().toMap() String start = wf['start'] @@ -193,7 +209,7 @@ ${meta['workflow']['stats']['processes']} wf.remove('complete') wf.remove('workflowStats') wf.remove('commandLine') - //printMap(wf, 'workflow') + // printMap(wf, 'workflow') log.info("\npublishing: ${wf['runName']}") } return [ @@ -209,7 +225,7 @@ ${meta['workflow']['stats']['processes']} String setupReadme() { String text = 'Stub README' try { - text = readme() + text = makeReadme() } catch (Exception e) { log.error("setupReadme failed: ${e.getMessage()}", pkg.meta) @@ -221,15 +237,27 @@ ${meta['workflow']['stats']['processes']} return text } - String readme() { + String makeReadme() { if (shouldSkip(KEY_README)) { log.info("readme=SKIP for ${pkg}") return null } GStringTemplateEngine engine = new GStringTemplateEngine() String raw_readme = pkg.meta_overrides(KEY_README, DEFAULT_README) + String cmd = "${meta['cmd']}".replace(' -', ' \\\n -') + String nf = meta['workflow']['nextflow'] + String nextflow = nf.replace(', ', '\\\n -')\ + .replace('nextflow.NextflowMeta(', ' \\\n -')\ + .replace(')', '') //log.debug("readme: ${raw_readme}") - Writable template = engine.createTemplate(raw_readme).make([meta: meta, msg: msg, now: now()]) + Writable template = engine.createTemplate(raw_readme).make([ + cmd: cmd, + meta: meta, + msg: msg, + nextflow: nextflow, + now: now(), + pkg: pkg.toString(), + ]) return template.toString() } From 31cf8325e2e13ccc27e7778140c9cfbc4605bc46 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Tue, 3 Oct 2023 16:53:09 -0700 Subject: [PATCH 8/8] handle missing meta --- .../main/nextflow/quilt/QuiltProduct.groovy | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index 66bc3a78..5bc876ab 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -66,17 +66,15 @@ ${cmd} ### Workflow -- workflow run name: ```${meta['workflow']?.get('runName')}``` -- scriptFile: ```${meta['workflow']?.get('scriptFile')}``` -- sessionId: ```${meta['workflow']?.get('sessionId')}``` -- start: ```${meta['time_start']}``` -- complete: ```${meta['time_complete']}``` +- **workflow run name**: ```${meta['workflow']?.get('runName')}``` +- **scriptFile**: ```${meta['workflow']?.get('scriptFile')}``` +- **sessionI**: ```${meta['workflow']?.get('sessionId')}``` +- **start**: ```${meta['time_start']}``` +- **complete**: ```${meta['time_complete']}``` ### Nextflow -```bash ${nextflow} -``` ### Processes @@ -245,20 +243,21 @@ ${nextflow} GStringTemplateEngine engine = new GStringTemplateEngine() String raw_readme = pkg.meta_overrides(KEY_README, DEFAULT_README) String cmd = "${meta['cmd']}".replace(' -', ' \\\n -') - String nf = meta['workflow']['nextflow'] - String nextflow = nf.replace(', ', '\\\n -')\ - .replace('nextflow.NextflowMeta(', ' \\\n -')\ - .replace(')', '') - //log.debug("readme: ${raw_readme}") - Writable template = engine.createTemplate(raw_readme).make([ + String nf = meta['workflow']?['nextflow'] + String nextflow = nf?.replace(', ', '```\n - **')\ + ?.replace('nextflow.NextflowMeta(', ' - **')\ + ?.replace(')', '```') + ?.replace(':', '**: ```') + String template = engine.createTemplate(raw_readme).make([ cmd: cmd, meta: meta, msg: msg, nextflow: nextflow, now: now(), - pkg: pkg.toString(), - ]) - return template.toString() + pkg: pkg.packageName, + ]).toString() + log.debug("readme.template: ${template}") + return template } List match(String glob) throws IOException {