From 847d29faf2ce5b40943348a52880bd46bf65a74d Mon Sep 17 00:00:00 2001 From: "Dr. Ernie Prabhakar" Date: Wed, 23 Oct 2024 21:44:11 -0700 Subject: [PATCH] feat(244) dynamic uris (#248) #244 --------- Co-authored-by: Dr. Ernie Prabhakar <19791+drernie@users.noreply.github.com> --- .github/workflows/mega-linter.yml | 6 +- .groovylintrc.json | 3 + CHANGELOG.md | 6 + Makefile | 8 +- README.md | 4 +- build.gradle | 3 +- gradle-groovysh-init.gradle | 98 +++++----- groovysh-task.gradle | 106 ++++++----- plugins/build.gradle | 26 +-- plugins/nf-quilt/build.gradle | 24 +-- .../main/nextflow/quilt/QuiltObserver.groovy | 171 +++--------------- .../main/nextflow/quilt/QuiltPathify.groovy | 147 +++++++++++++++ .../main/nextflow/quilt/QuiltProduct.groovy | 73 ++++---- .../nextflow/quilt/jep/QuiltPackage.groovy | 26 ++- .../nextflow/quilt/jep/QuiltParser.groovy | 7 +- .../quilt/nio/QuiltFileAttributes.groovy | 2 +- .../quilt/nio/QuiltFileSystemProvider.groovy | 6 +- .../src/resources/META-INF/MANIFEST.MF | 2 +- .../nextflow/quilt/QuiltObserverTest.groovy | 135 +------------- .../nextflow/quilt/QuiltPathifyTest.groovy | 126 +++++++++++++ .../test/nextflow/quilt/QuiltPkgTest.groovy | 2 - .../nextflow/quilt/QuiltProductTest.groovy | 106 ++++++++--- .../nextflow/quilt/QuiltSpecification.groovy | 10 +- .../quilt/jep/QuiltPackageTest.groovy | 5 - .../nextflow/quilt/jep/QuiltParserTest.groovy | 1 - .../nio/QuiltFileSystemProviderTest.groovy | 5 +- .../nextflow/quilt/nio/QuiltNioTest.groovy | 1 - .../nextflow/quilt/nio/QuiltPathTest.groovy | 2 +- settings.gradle | 2 +- 29 files changed, 608 insertions(+), 505 deletions(-) create mode 100644 plugins/nf-quilt/src/main/nextflow/quilt/QuiltPathify.groovy create mode 100644 plugins/nf-quilt/src/test/nextflow/quilt/QuiltPathifyTest.groovy diff --git a/.github/workflows/mega-linter.yml b/.github/workflows/mega-linter.yml index 6bee44f7..f24067b7 100644 --- a/.github/workflows/mega-linter.yml +++ b/.github/workflows/mega-linter.yml @@ -6,14 +6,16 @@ on: # Trigger mega-linter at every push. Action will also be visible from Pull Requests to main push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions) -permissions: read-all +permissions: + issues: write + pull-requests: write env: # Comment env block if you do not want to apply fixes # Apply linter fixes configuration APPLY_FIXES: all # When active, APPLY_FIXES must also be defined as environment variable (in github/workflows/mega-linter.yml or other CI tool) #APPLY_FIXES_EVENT: pull_request # Decide which event triggers application of fixes in a commit or a PR (pull_request, push, all) #APPLY_FIXES_MODE: pull_request # If APPLY_FIXES is used, defines if the fixes are directly committed (commit) or posted in a PR (pull_request) - DISABLE_LINTERS: SPELL_CSPELL,COPYPASTE_JSCPD,REPOSITORY_GITLEAKS + DISABLE_LINTERS: SPELL_CSPELL,COPYPASTE_JSCPD,REPOSITORY_GITLEAKS,GROOVY_NPM_GROOVY_LINT FILTER_REGEX_EXCLUDE: .*/.*gradle concurrency: diff --git a/.groovylintrc.json b/.groovylintrc.json index efe62778..536acf30 100644 --- a/.groovylintrc.json +++ b/.groovylintrc.json @@ -28,6 +28,9 @@ "JUnitPublicNonTestMethod": { "enabled": false }, + "JUnitTestMethodWithoutAssert": { + "enabled": false + }, "JavaIoPackageAccess": { "enabled": false }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 51450b2d..2fbabcf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [0.8.7] 2024-10-23 + +- Use package cache instead of `params` to find output URIs + (in order to support dynamic URIs set by, e.g. `main.nf`) +- Allow setting metadata from inside the workflow + ## [0.8.6] 2024-09-11 - Fix addOverlay bug on subfolders diff --git a/Makefile b/Makefile index 964c9d16..0f23079c 100644 --- a/Makefile +++ b/Makefile @@ -16,11 +16,11 @@ REPORT ?= ./plugins/$(PROJECT)/build/reports/tests/test/index.html verify: #compile echo $(WRITE_BUCKET) - ./gradlew check || open $(REPORT) + ./gradlew test ${ONE} || open $(REPORT) fast: - ./gradlew ${mm}test --fail-fast || open ./plugins/nf-quilt/build/reports/tests/test/index.html - + ./gradlew test ${ONE} --fail-fast || open $(REPORT) +# example: make fast ONE="--tests QuiltProductTest" check-env: echo $(VERSION) echo $(WRITE_BUCKET) @@ -38,7 +38,7 @@ compile: ./gradlew compileGroovy exportClasspath @echo "DONE `date`" -nextflow-git: +nextflow: if [ ! -d "$(NF_DIR)" ]; then git clone https://github.com/nextflow-io/nextflow.git "$(NF_DIR)"; fi cd "$(NF_DIR)"; git checkout && make compile && git restore .; cd .. diff --git a/README.md b/README.md index ff322e63..f9034887 100644 --- a/README.md +++ b/README.md @@ -93,8 +93,8 @@ From the command-line, do, e.g.: ```bash # export NXF_VER=23.04.3 export LOG4J_DEBUG=true # for verbose logging -export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.8.6/nf-quilt-0.8.6-meta.json -nextflow run main.nf -plugins nf-quilt@0.8.6 +export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.8.7/nf-quilt-0.8.7-meta.json +nextflow run main.nf -plugins nf-quilt@0.8.7 ``` For Tower, you can use the "Pre-run script" to set the environment variables. diff --git a/build.gradle b/build.gradle index 4845d93b..d2c862f8 100644 --- a/build.gradle +++ b/build.gradle @@ -194,9 +194,8 @@ def getRuntimeConfigs() { task exportClasspath { dependsOn allprojects.jar doLast { - def home = System.getProperty('user.home') def all = getRuntimeConfigs() - def libs = all.collect { File file -> /*println file.canonicalPath.replace(home, '$HOME');*/ file.canonicalPath; } + def libs = all.collect { File file -> file.canonicalPath; } ['nextflow', 'nf-commons', 'nf-httfs'].each { libs << file("modules/$it/build/libs/${it}-${version}.jar").canonicalPath } file('.launch.classpath').text = libs.unique().join(':') } diff --git a/gradle-groovysh-init.gradle b/gradle-groovysh-init.gradle index f10c6575..4c36259c 100644 --- a/gradle-groovysh-init.gradle +++ b/gradle-groovysh-init.gradle @@ -1,49 +1,49 @@ -gradle.projectsLoaded { - rootProject { - afterEvaluate { project -> - if (!project.repositories.any{it.name == 'MavenRepo'}) { - project.repositories { - // To be able to load org.codehaus.groovy:groovy-groovysh - mavenCentral() - } - } - - project.configurations { - groovyshdependencies - } - - project.dependencies { - groovyshdependencies("org.codehaus.groovy:groovy-groovysh:${GroovySystem.version}") { - exclude group: 'org.codehaus.groovy' - } - } - - project.tasks.register('groovysh') { - group 'debug' - description 'Runs an interactive shell in the context of the project.' - doLast { - URLClassLoader groovyObjectClassLoader = GroovyObject.class.classLoader - def groovyshClass - def groovyShell - - // Add dependency jars to classloader - configurations.groovyshdependencies.each {File file -> - groovyObjectClassLoader.addURL(file.toURL()) - } - Class.forName('jline.console.history.FileHistory', true, groovyObjectClassLoader) - groovyshClass = Class.forName('org.codehaus.groovy.tools.shell.Groovysh', true, groovyObjectClassLoader) - - if (groovyshClass) { - groovyShell = groovyshClass.newInstance() - } - if (groovyShell) { - groovyShell.interp.context.variables.put("gradle", gradle) - groovyShell.interp.context.variables.put("settings", gradle.settings) - groovyShell.interp.context.variables.put("project", project) - groovyShell.run('') - } - } - } - } - } -} \ No newline at end of file +gradle.projectsLoaded { + rootProject { + afterEvaluate { project -> + if (!project.repositories.any { it.name == 'MavenRepo' }) { + project.repositories { + // To be able to load org.codehaus.groovy:groovy-groovysh + mavenCentral() + } + } + + project.configurations { + groovyshdependencies + } + + project.dependencies { + groovyshdependencies("org.codehaus.groovy:groovy-groovysh:${GroovySystem.version}") { + exclude group: 'org.codehaus.groovy' + } + } + + project.tasks.register('groovysh') { + group 'debug' + description 'Runs an interactive shell in the context of the project.' + doLast { + URLClassLoader groovyObjectClassLoader = GroovyObject.classLoader + def groovyshClass + def groovyShell + + // Add dependency jars to classloader + configurations.groovyshdependencies.each { File file -> + groovyObjectClassLoader.addURL(file.toURL()) + } + Class.forName('jline.console.history.FileHistory', true, groovyObjectClassLoader) + groovyshClass = Class.forName('org.codehaus.groovy.tools.shell.Groovysh', true, groovyObjectClassLoader) + + if (groovyshClass) { + groovyShell = groovyshClass.newInstance() + } + if (groovyShell) { + groovyShell.interp.context.variables.put('gradle', gradle) + groovyShell.interp.context.variables.put('settings', gradle.settings) + groovyShell.interp.context.variables.put('project', project) + groovyShell.run('') + } + } + } + } +} +} diff --git a/groovysh-task.gradle b/groovysh-task.gradle index 703750c3..b2576942 100644 --- a/groovysh-task.gradle +++ b/groovysh-task.gradle @@ -1,54 +1,52 @@ -gradle.projectsLoaded { - rootProject { - afterEvaluate { project -> - if (!project.repositories.any{it.name == 'MavenRepo'}) { - project.repositories { - // To be able to load org.apache.groovy:groovy-groovysh and dependencies - mavenCentral { - content { - includeGroup 'org.apache.groovy' - includeGroup 'jline' - includeGroup 'com.github.javaparser' - includeGroup 'org.ow2.asm' - includeGroup 'org.abego.treelayout' - includeGroup 'org.apache.ivy' - } - } - } - } - project.configurations { - groovyshdependencies - } - - project.dependencies { - groovyshdependencies "org.apache.groovy:groovy-groovysh:4.0.23" - } - - project.tasks.register('groovysh') { - group 'debug' - description 'Runs an interactive shell in the context of the project. Use :inspect command to inspect project, gradle, settings or other objects.' - doLast { - URLClassLoader groovyshClassLoader = new URLClassLoader(); - configurations.groovyshdependencies.each {File file -> - groovyshClassLoader.addURL(file.toURI().toURL()) - } - - def fileHistoryClass - def groovyshClass - def groovyShell - fileHistoryClass = Class.forName('jline.console.history.FileHistory', true, groovyshClassLoader) - groovyshClass = Class.forName('org.apache.groovy.groovysh.Groovysh', true, groovyshClassLoader) - if (groovyshClass) { - groovyShell = groovyshClass.newInstance() - if (groovyShell) { - groovyShell.interp.context.variables.put("gradle", gradle) - groovyShell.interp.context.variables.put("settings", gradle.settings) - groovyShell.interp.context.variables.put("project", project) - groovyShell.run('# Available objects: gradle, settings, project\n# Try :inspect project') - } - } - } - } - } - } -} +gradle.projectsLoaded { + rootProject { + afterEvaluate { project -> + if (!project.repositories.any { it.name == 'MavenRepo' }) { + project.repositories { + // To be able to load org.apache.groovy:groovy-groovysh and dependencies + mavenCentral { + content { + includeGroup 'org.apache.groovy' + includeGroup 'jline' + includeGroup 'com.github.javaparser' + includeGroup 'org.ow2.asm' + includeGroup 'org.abego.treelayout' + includeGroup 'org.apache.ivy' + } + } + } + } + project.configurations { + groovyshdependencies + } + + project.dependencies { + groovyshdependencies 'org.apache.groovy:groovy-groovysh:4.0.23' + } + + project.tasks.register('groovysh') { + group 'debug' + description 'Runs an interactive shell in the context of the project. Use :inspect command to inspect project, gradle, settings or other objects.' + doLast { + URLClassLoader groovyshClassLoader = new URLClassLoader() + configurations.groovyshdependencies.each { File file -> + groovyshClassLoader.addURL(file.toURI().toURL()) + } + + def groovyshClass + def groovyShell + groovyshClass = Class.forName('org.apache.groovy.groovysh.Groovysh', true, groovyshClassLoader) + if (groovyshClass) { + groovyShell = groovyshClass.newInstance() + if (groovyShell) { + groovyShell.interp.context.variables.put('gradle', gradle) + groovyShell.interp.context.variables.put('settings', gradle.settings) + groovyShell.interp.context.variables.put('project', project) + groovyShell.run('# Available objects: gradle, settings, project\n# Try :inspect project') + } + } + } + } + } +} +} diff --git a/plugins/build.gradle b/plugins/build.gradle index ed48d2c5..ea8f6e96 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -15,9 +15,9 @@ */ plugins { - id "java" - id "groovy" - id "io.nextflow.nf-build-plugin" version "1.0.1" + id 'java' + id 'groovy' + id 'io.nextflow.nf-build-plugin' version '1.0.1' } ext.github_organization = project.findProperty('github_organization') ?: 'nextflow-io' @@ -29,8 +29,9 @@ ext.github_index_url = "https://github.com/${github_organization}/plugins/main/p jar.enabled = false String computeSha512(File file) { - if( !file.exists() ) + if (!file.exists()) { throw new GradleException("Missing file: $file -- cannot compute SHA-512") + } return org.apache.commons.codec.digest.DigestUtils.sha512Hex(file.bytes) } @@ -40,7 +41,7 @@ String now() { List allPlugins() { def plugins = [] - new File(rootProject.rootDir, 'plugins') .eachDir { if(it.name.startsWith('nf-')) plugins.add(it.name) } + new File(rootProject.rootDir, 'plugins') .eachDir { if (it.name.startsWith('nf-')) plugins.add(it.name) } return plugins } @@ -48,7 +49,7 @@ String metaFromManifest(String meta, File file) { def str = file.text def regex = ~/(?m)^$meta:\s*([\w-\.<>=]+)$/ def m = regex.matcher(str) - if( m.find() ) { + if (m.find()) { def ver = m.group(1) println "Set plugin '${file.parentFile.parentFile.parentFile.parentFile.name}' version=${ver}" return ver @@ -68,7 +69,7 @@ subprojects { mavenCentral() } - version = metaFromManifest('Plugin-Version',file('src/resources/META-INF/MANIFEST.MF')) + version = metaFromManifest('Plugin-Version', file('src/resources/META-INF/MANIFEST.MF')) tasks.withType(Jar) { duplicatesStrategy = DuplicatesStrategy.INCLUDE @@ -94,7 +95,7 @@ subprojects { "version": "${project.version}", "date": "${timestamp}", "url": "https://github.com/${github_organization}/${project.name}/releases/download/${project.version}/${project.name}-${project.version}.zip", -"requires": "${metaFromManifest('Plugin-Requires',file('src/resources/META-INF/MANIFEST.MF'))}", +"requires": "${metaFromManifest('Plugin-Requires', file('src/resources/META-INF/MANIFEST.MF'))}", "sha512sum": "${computeSha512(zip)}" } """ @@ -129,10 +130,10 @@ subprojects { /* * "install" the plugin the project root build/plugins directory */ - project.parent.tasks.getByName("assemble").dependsOn << copyPluginZip + project.parent.tasks.getByName('assemble').dependsOn << copyPluginZip task uploadPlugin(type: io.nextflow.gradle.tasks.GithubUploader, dependsOn: makeZip) { - assets = providers.provider {["$buildDir/libs/${project.name}-${project.version}.zip", + assets = providers.provider { ["$buildDir/libs/${project.name }-${project.version }.zip", "$buildDir/libs/${project.name}-${project.version}-meta.json" ]} release = providers.provider { project.version } repo = providers.provider { project.name } @@ -141,7 +142,6 @@ subprojects { authToken = github_access_token skipExisting = true } - } task upload(dependsOn: [subprojects.uploadPlugin]) { } @@ -151,11 +151,11 @@ classes.dependsOn subprojects.copyPluginLibs /* * Merge and publish the plugins index file */ -task publishIndex( type: io.nextflow.gradle.tasks.GithubRepositoryPublisher ) { +task publishIndex(type: io.nextflow.gradle.tasks.GithubRepositoryPublisher) { indexUrl = github_index_url repos = allPlugins() owner = github_organization githubUser = github_username githubEmail = github_commit_email githubToken = github_access_token -} \ No newline at end of file +} diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index afde3588..e84db762 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -24,8 +24,8 @@ plugins { } useLatestVersions { - // A blacklist of dependencies to update, in the format of group:name - updateBlacklist = [ + // A blacklist of dependencies to update, in the format of group:name + updateBlacklist = [ 'com.quiltdata:quiltcore', 'org.codehaus.groovy:groovy', 'org.codehaus.groovy:groovy-nio', @@ -92,22 +92,22 @@ dependencies { //testImplementation(testFixtures('black.ninia:jep:4.0.3')) // test configuration - testImplementation "org.codehaus.groovy:groovy:3.0.22" - testImplementation "org.codehaus.groovy:groovy-nio:3.0.22" + testImplementation 'org.codehaus.groovy:groovy:3.0.22' + testImplementation 'org.codehaus.groovy:groovy-nio:3.0.22' testImplementation "io.nextflow:nextflow:$nextflowVersion" - testImplementation ("org.codehaus.groovy:groovy-test:3.0.22") { exclude group: 'org.codehaus.groovy' } - testImplementation ("cglib:cglib-nodep:3.3.0") - testImplementation ("org.objenesis:objenesis:3.4") - testImplementation ("org.spockframework:spock-core:2.3-groovy-3.0") { exclude group: 'org.codehaus.groovy'; exclude group: 'net.bytebuddy' } - testImplementation ('org.spockframework:spock-junit4:2.3-groovy-3.0') { exclude group: 'org.codehaus.groovy'; exclude group: 'net.bytebuddy' } - testImplementation ('com.google.jimfs:jimfs:1.3.0') + testImplementation('org.codehaus.groovy:groovy-test:3.0.22') { exclude group: 'org.codehaus.groovy' } + testImplementation('cglib:cglib-nodep:3.3.0') + testImplementation('org.objenesis:objenesis:3.4') + testImplementation('org.spockframework:spock-core:2.3-groovy-3.0') { exclude group: 'org.codehaus.groovy'; exclude group: 'net.bytebuddy' } + testImplementation('org.spockframework:spock-junit4:2.3-groovy-3.0') { exclude group: 'org.codehaus.groovy'; exclude group: 'net.bytebuddy' } + testImplementation('com.google.jimfs:jimfs:1.3.0') testImplementation(testFixtures("io.nextflow:nextflow:$nextflowVersion")) testImplementation(testFixtures("io.nextflow:nf-commons:$nextflowVersion")) // see https://docs.gradle.org/4.1/userguide/dependency_management.html#sec:module_replacement modules { - module("commons-logging:commons-logging") { replacedBy("org.slf4j:jcl-over-slf4j") } + module('commons-logging:commons-logging') { replacedBy('org.slf4j:jcl-over-slf4j') } } } @@ -125,7 +125,7 @@ jacocoTestCoverageVerification { violationRules { rule { limit { - minimum = 0.7 + minimum = 0.65 } } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy index 5381fbd9..82b3f8b7 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy @@ -16,10 +16,6 @@ package nextflow.quilt import nextflow.Session -import nextflow.quilt.jep.QuiltParser -import nextflow.quilt.jep.QuiltPackage -import nextflow.quilt.nio.QuiltPath -import nextflow.quilt.nio.QuiltPathFactory import nextflow.trace.TraceObserver import java.nio.file.Path @@ -41,145 +37,35 @@ class QuiltObserver implements TraceObserver { private Session session private String workDir - // Is this overkill? Do we only ever have one output URI per run? - private String[] outputPrefixes = ['pub', 'out'] - final private Map outputURIs = [:] - final private Map> packageOverlays = [:] + final private Map configMetadata = [:] final private Lock lock = new ReentrantLock() // Need this because of threads - - static QuiltPath asQuiltPath(Path path) { - if (path in QuiltPath) { - return (QuiltPath) path - } - String strPath = path.getFileName() - if (strPath.contains('#package')) { - String url = "${QuiltParser.SCHEME}://${strPath}" - return QuiltPathFactory.parse(url) - } - return null - } - - static String quiltURIfromS3(String s3uri) { - log.debug("quiltURIfromS3: $s3uri") - String[] partsArray = s3uri.split('/') - List parts = new ArrayList(partsArray.toList()) - // parts.eachWithIndex { p, i -> println("quiltURIfromS3.parts[$i]: $p") } - - if (parts.size() < 2) { - throw new IllegalArgumentException("Invalid s3uri[${parts.size()}]: $parts") - } - parts = parts.drop(2) - if (parts[0].endsWith(':')) { - parts = parts.drop(1) - } - String bucket = parts.remove(0) - String dest = parts.join('%2f') - String suffix = parts.size() > 1 ? parts.removeLast() : 'default_suffix' - String prefix = parts.size() > 0 ? parts.removeLast() : 'default_prefix' - String base = "quilt+s3://${bucket}#package=${prefix}%2f${suffix}" - String uri = base + '&dest=' + ((dest) ?: '/') - return uri - } - - static String pkgKey(QuiltPath path) { - return QuiltPackage.osConvert("${path.getBucket()}/${path.getPackageName()}") - } - - void findOutputParams(Map params) { - log.debug("findOutputParams[$params]") - params.each { key, value -> - String uri = "$value" - if (outputPrefixes.any { key.startsWith(it) && !key.contains('-') }) { - String[] splits = uri.split(':') - if (splits.size() < 2) { - log.debug("Unrecognized URI[$uri] for key[$key] matching $outputPrefixes") - return - } - String scheme = splits[0] - if (scheme == 's3') { - uri = quiltURIfromS3(uri) - } else if (scheme != 'quilt+s3') { - log.warn("Unrecognized scheme:$scheme for output URI[$key]: $uri") - return - } - QuiltPath path = QuiltPathFactory.parse(uri) - String pkgKey = pkgKey(path) - outputURIs[pkgKey] = uri - } - } - } + // Is this overkill? Do we ever have more than one output URI per run? + final private Map publishedPaths = [:] void checkConfig(Map> config) { - Object prefixes = config.get('quilt')?.get('outputPrefixes') - if (prefixes) { - outputPrefixes = prefixes as String[] + Object metadata = config.get('quilt')?.get('metadata') + if (metadata) { + configMetadata.putAll(metadata as Map) } } - String workRelative(Path src) { - Path source = src.toAbsolutePath().normalize() - Path workDir = session.workDir.toAbsolutePath().normalize() - try { - Path subPath = workDir.relativize(source) - // drop first two components, which are the workDir - Path relPath = subPath.subpath(2, subPath.getNameCount()) - return relPath.toString() - } catch (IllegalArgumentException e) { - log.error("workRelative.fallback: $e") - log.warn("Cannot relativize source:${source.getClass()} to workDir:${workDir.getClass()}") - return source.toString() + boolean checkExtractedPath(QuiltPathify pathify) { + String key = pathify.pkgKey() + if (key in publishedPaths) { + return true } + log.debug("checkExtractedPath: $key not in publishedPaths") + addPublishedPath(key, pathify) + return false } - String pkgRelative(String key, Path dest) { - String destString = QuiltPackage.osConvert(dest.toAbsolutePath().normalize().toString()) - String pkgKey = QuiltPackage.osConvert(key) - // find pkgKey in destination.toString() - int index = destString.indexOf(pkgKey) - println("pkgRelative[$index]: $pkgKey in $destString") - // return the portion after the end of pkgKey - int len = index + pkgKey.length() + 1 - if (index >= 0 && len < destString.length()) { - return destString.substring(len) - } - return null - } - - String addOverlay(String pkgKey, Path dest, Path source) { + void addPublishedPath(String key, QuiltPathify pathify) { lock.lock() try { - Map overlays = packageOverlays.get(pkgKey, [:]) as Map - String relPath = pkgRelative(pkgKey, dest) - println("addOverlay.relPath: $relPath") - log.debug("addOverlay[$relPath] = dest:$dest <= source:$source") - overlays[relPath] = source - packageOverlays[pkgKey] = overlays - return relPath + publishedPaths[key] = pathify } finally { lock.unlock() } - return null - } - - boolean confirmQuiltPath(QuiltPath qPath) { - log.debug("confirmQuiltPath[$qPath]") - String key = pkgKey(qPath) - log.debug("confirmQuiltPath: key[$key] in outputURIs[${outputURIs.size()}]: $outputURIs") - return outputURIs.containsKey(key) ? true : false - } - - boolean canOverlayPath(Path dest, Path source) { - log.debug("canOverlayPath[$dest] <- $source") - Set keys = outputURIs.keySet() - for (String key : keys) { - if (dest.toString().contains(key)) { - log.debug("canOverlayPath: matched key[$key] to $dest") - addOverlay(key, dest, source) - return true - } - } - log.error("canOverlayPath: no key found for $dest in $keys") - return false } @Override @@ -187,7 +73,6 @@ class QuiltObserver implements TraceObserver { log.debug("`onFlowCreate` $this") this.session = session this.workDir = session.config.workDir - findOutputParams(session.getParams()) checkConfig(session.config) } @@ -195,27 +80,25 @@ class QuiltObserver implements TraceObserver { void onFilePublish(Path destination, Path source) { // Path source may be null, won't work with older versions of Nextflow log.debug("onFilePublish.Path[$destination] <- $source") - if (!outputURIs) { - // NOTE: TraceFileObserver calls onFilePublish _before_ onFlowCreate - log.debug('onFilePublish: no outputURIs yet') + if (!session) { + log.debug('onFilePublish: no session intialized') return } - QuiltPath qPath = asQuiltPath(destination) - boolean ok = (qPath != null) ? confirmQuiltPath(qPath) : canOverlayPath(destination, source) - if (!ok) { - log.error("onFilePublish: no match for $destination") + QuiltPathify pathify = new QuiltPathify(destination) + if (pathify.isOverlay && source == null) { + log.error("onFilePublish.isOverlay: no source for $pathify") + return } + checkExtractedPath(pathify) } @Override void onFlowComplete() { - log.debug("onFlowComplete.outputURIs[${outputURIs.size()}]: $outputURIs") - // create QuiltProduct for each unique package URI - outputURIs.each { key, uri -> - QuiltPath path = QuiltPathFactory.parse(uri) - Map overlays = packageOverlays.get(key, [:]) as Map - log.debug("onFlowComplete.pkg: $path overlays[${overlays?.size()}]: $overlays") - new QuiltProduct(path, session, overlays) + log.debug("onFlowComplete.publishedPaths[${publishedPaths.size()}]: $publishedPaths") + // create a QuiltProduct for each unique package key + publishedPaths.each { key, path -> + log.debug("onFlowComplete: $key -> $path") + new QuiltProduct(path, session) } } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltPathify.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltPathify.groovy new file mode 100644 index 00000000..26dde791 --- /dev/null +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltPathify.groovy @@ -0,0 +1,147 @@ +/* + * Copyright 2022, Quilt Data Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nextflow.quilt + +import nextflow.quilt.jep.QuiltParser +import nextflow.quilt.jep.QuiltPackage +import nextflow.quilt.nio.QuiltPath +import nextflow.quilt.nio.QuiltPathFactory + +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.Files + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j + +/** + * Extracts QuiltPath objects from published Path + * + * @author Ernest Prabhakar + */ +@Slf4j +@CompileStatic +class QuiltPathify { + + boolean isOverlay = false + QuiltPath path + QuiltPackage pkg + String uri + + static void copyFile(Path source, String destRoot, String relpath) { + Path dest = Paths.get(destRoot, relpath.split('/') as String[]) + try { + dest.getParent().toFile().mkdirs() // ensure directories exist first + Files.copy(source, dest) + } + catch (Exception e) { + log.error("writeString: cannot write `$source` to `$dest` in `${destRoot}`") + } + } + + /** + * Converts an S3 path to a Quilt URI. + * + * This method takes an absolute path representing an S3 output uri + * and extracts the bucket, prefix, suffix, and path components + * to construct a Quilt URI. + * + * Example input: + * 
+     * /udp-spec/nf-quilt/s3-test/inputs/a_folder/THING_TWO.md
+     *
+ * Extracted components: + * - bucket: udp-spec + * - prefix: nf-quilt (or `default_prefix` if missing) + * - suffix: s3-test (or `default_suffix` if missing) + * - path: inputs/a_folder/THING_TWO.md + */ + static String uriFromS3File(String s3path) { + log.debug("uriFromS3File: $s3path") + String[] partsArray = s3path.split('/') + List parts = new ArrayList(partsArray.toList()) + // parts.eachWithIndex { p, i -> println("uriFromS3File.parts[$i]: $p") } + if (parts.size() < 2) { + log.error("uriFromS3File: invalid path: $s3path") + return '' + } + parts.remove(0) // remove leading slash + String file = parts.remove(parts.size() - 1) + + String bucket = parts.size() > 0 ? parts.remove(0) : QuiltParser.NULL_BUCKET + String prefix = parts.size() > 0 ? parts.remove(0) : 'default_prefix' + String suffix = parts.size() > 0 ? parts.remove(0) : 'default_suffix' + String folder = parts.join('/') + String sub_path = folder.length() > 0 ? folder + '/' + file : file + + log.debug("uriFromS3File: $bucket/$prefix/$suffix/$sub_path") + String base = "quilt+s3://${bucket}#package=${prefix}%2f${suffix}" + String uri = base + '&path=' + sub_path + return uri + } + + // Constructor takes a Path and finds QuiltPath and QuiltPackage + QuiltPathify(Path path) { + if (path in QuiltPath) { + this.path = (QuiltPath) path + this.uri = this.path.toUriString() + this.pkg = this.path.pkg() + } else if (!findQuiltPath(path.getFileName().toString())) { + makeQuiltPath(path) + this.isOverlay = true + } + } + + boolean findQuiltPath(String filename) { + // check for '#package' in filename + if (!filename.toString().contains('#package')) { + return false + } + + uri = "${QuiltParser.SCHEME}://${filename}" + path = QuiltPathFactory.parse(this.uri) + pkg = path.pkg() + return true + } + + boolean makeQuiltPath(Path path) { + String quiltURI = uriFromS3File(path.toString()) + this.path = QuiltPathFactory.parse(quiltURI) + this.uri = this.path.toUriString() + this.pkg = this.path.pkg() + return true + } + + boolean copyToPackage(Path source) { + if (!this.isOverlay) { + return false + } + String localPath = path.sub_paths() + Path destDir = pkg.packageDest() + log.debug("copyToPackage: $source -> $destDir / $localPath") + copyFile(source, destDir.toString(), localPath) + return true + } + + String pkgKey() { + return pkg.toKey() + } + + String toString() { + return "QuiltPathify[${uri}]" + } + +} diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index a82f7559..4af9d431 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -78,7 +78,7 @@ ${nextflow} ### Processes -`${meta['workflow']['stats']['processes']}` +`${meta['workflow']?.get('stats')?.get('processes')}` ''' private final static String DEFAULT_SUMMARIZE = '*.md,*.html,*.?sv,*.pdf,igv.json,**/multiqc_report.html' @@ -119,49 +119,32 @@ ${nextflow} static String now() { LocalDateTime time = LocalDateTime.now() - return time.toString() + return time.toString().replace(':', '-').replace('T', 't') } private final QuiltPath path - private final List overlays private final QuiltPackage pkg private final Session session private String msg private Map meta - QuiltProduct(QuiltPath path, Session session, Map overlays = [:]) { - this.path = path - this.pkg = path.pkg() + QuiltProduct(QuiltPathify pathify, Session session) { + this.path = pathify.path + this.pkg = pathify.pkg this.msg = pkg.toString() - this.meta = [pkg: msg, time_start: now()] + this.meta = pkg.meta + [pkg: msg, time_start: now()] this.session = session if (session.isSuccess() || pkg.is_force()) { - if (overlays) { - log.debug("publishing overlays: ${overlays.size()}") - publishOverlays(overlays) - } else { - log.info('No overlays to publish.') - } publish() } else { log.info("not publishing: ${pkg} [unsuccessful session]") } } - void publishOverlays(Map overlays) { - /// Copying published files to inside package directory - /// for (re)upload to the package - /// FIXME: Replace this with in-place packaging - overlays.each { relpath, source -> - log.info("publishing overlay[$relpath]: ${source}") - copyFile(source, pkg.packageDest().toString(), relpath) - } - } - void publish() { log.debug("publish($msg)") - meta = setupMeta() + addSessionMeta() setupReadme() setupSummarize() try { @@ -180,21 +163,28 @@ ${nextflow} } boolean shouldSkip(key) { + print("shouldSkip[$key]: ${pkg.meta}") return pkg.meta.containsKey(key) && pkg.meta[key] == KEY_SKIP } - Map setupMeta() { - try { - meta = getMetadata(session.config) - meta['quilt'] = [package_id: pkg.toString(), uri: path.toUriString()] - msg = "${meta['config']['runName']}: ${meta['cmd']}" - meta.remove('config') + boolean addSessionMeta() { + if (shouldSkip(KEY_META)) { + return false } - catch (Exception e) { - log.error("setupMeta failed: ${e.getMessage()}", pkg.meta) + try { + Map smeta = getMetadata(session.config) + println("addSessionMeta.smeta: ${smeta}") + smeta['quilt'] = [package_id: pkg.toString(), uri: path.toUriString()] + msg = "${smeta['config']['runName']}: ${smeta['cmd']}" + smeta.remove('config') + meta += smeta + } catch (Exception e) { + log.error("addSessionMeta.getMetadata failed: ${e.getMessage()}", pkg.meta) + return false } + println("addSessionMeta.meta: ${meta}") writeNextflowMetadata(meta, 'metadata') - return shouldSkip(KEY_META) ? [:] : meta + return true } String writeNextflowMetadata(Map map, String suffix) { @@ -205,6 +195,7 @@ ${nextflow} } Map getMetadata(Map cf) { + // add metadata from quilt and URI if (cf != null) { cf.remove('executor') cf.remove('params') @@ -218,7 +209,7 @@ ${nextflow} writeNextflowMetadata(params, 'params') params.remove('genomes') params.remove('test_data') - // printMap(params, 'params') + // printMap(params, 'params') } Map wf = session.getWorkflowMetadata().toMap() String start = wf['start'] @@ -251,7 +242,7 @@ ${nextflow} text = makeReadme() } catch (Exception e) { - log.error("setupReadme failed: ${e.getMessage()}", pkg.meta) + log.error("setupReadme failed: ${e.getMessage()}\n{$e}", pkg.meta) } if (text != null && text.length() > 0) { //log.debug("setupReadme: ${text.length()} bytes") @@ -273,15 +264,17 @@ ${nextflow} ?.replace('nextflow.NextflowMeta(', ' - **')\ ?.replace(')', '```') ?.replace(':', '**: ```') - String template = engine.createTemplate(raw_readme).make([ + Map params = [ cmd: cmd, meta: meta, msg: msg, nextflow: nextflow, now: now(), pkg: pkg.packageName, - ]) - // log.debug("readme.template: ${template}") + ] + log.debug("makeReadme.params: ${params}") + String template = engine.createTemplate(raw_readme).make(params) + log.debug("makeReadme.template: ${template}") return template } @@ -302,13 +295,13 @@ ${nextflow} matches.add(rel) } return FileVisitResult.CONTINUE - } + } @Override FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { return FileVisitResult.CONTINUE - } + } }) return matches diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index 893dff9a..500e15e2 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -97,13 +97,18 @@ class QuiltPackage { PKGS.clear() } + static QuiltPackage forUriString(String uri) { + QuiltParser parsed = QuiltParser.forUriString(uri) + return forParsed(parsed) + } + static QuiltPackage forParsed(QuiltParser parsed) { boolean isNull = parsed.hasNullBucket() if (isNull && !PKGS.isEmpty()) { return PKGS.values().last() } - String pkgKey = parsed.toPackageString() + String pkgKey = parsed.toPackageString(true) // ignore metadata for Key log.debug("QuiltPackage.forParsed[${pkgKey}]") def pkg = PKGS.get(pkgKey) if (pkg) { return pkg } @@ -113,6 +118,17 @@ class QuiltPackage { return pkg } + static boolean hasKey(String pkgKey) { + return PKGS.containsKey(pkgKey) + } + + static QuiltPackage forKey(String pkgKey) { + if (hasKey(pkgKey)) { + return PKGS.get(pkgKey) + } + return null + } + static List listDirectory(Path rootPath) { return Files.walk(rootPath).sorted(Comparator.reverseOrder()).collect(Collectors.toList()) } @@ -313,6 +329,14 @@ class QuiltPackage { return "QuiltPackage.${bucket}_${packageName}".replaceAll(/[-\/]/, '_') } + String toUriString() { + return parsed.toUriString() + } + + String toKey() { + return parsed.toPackageString(true) + } + String meta_overrides(String key, Serializable baseline = null) { Object temp = meta[key] ? meta[key] : baseline return temp.toString() diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltParser.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltParser.groovy index 8ea8802d..a9123569 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltParser.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltParser.groovy @@ -143,7 +143,7 @@ class QuiltParser { this.catalogName = options.get(P_CAT) this.options = options this.metadata = metadata - // log.debug("QuiltParser[${bucket}] for ${packageName} in ${path}") + // log.debug("QuiltParser[${bucket}] for ${packageName} in ${path}") } String parsePkg(String pkg) { @@ -281,9 +281,9 @@ class QuiltParser { return options?.get(key) } - String toPackageString() { + String toPackageString(boolean forKey = false) { String str = "${getBucket()}" - if (metadata) { + if (metadata && !forKey) { str += "?${unparseQuery(metadata)}" } if (packageName) { @@ -292,7 +292,6 @@ class QuiltParser { if (tag) { pkg += ":$tag" } str += "#package=${pkg.replace('/', '%2f')}" } - // log.debug("toPackageString: ${str}") return str } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileAttributes.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileAttributes.groovy index ab1ef34b..49a4f0a1 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileAttributes.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileAttributes.groovy @@ -40,7 +40,7 @@ class QuiltFileAttributes implements BasicFileAttributes { this.key = path.isJustPackage() ? '/' : path.file_key() this.origKey = key this.attrs = attrs - //log.debug("QuiltFileAttributes($path): this=$this") + //log.debug("QuiltFileAttributes($path): this=$this") } @Override diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy index 8165affc..5ed5ad7e 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy @@ -110,10 +110,6 @@ class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTr log.info "download Quilt package: ${pkg} installed? ${pkg.installed}" if (!pkg.installed) { Path dest = pkg.install() - if (!dest) { - log.error "download.install failed: ${pkg}" - throw new IOException("Failed to install Quilt package: ${pkg}") - } log.info "download.installed Quilt package to: $dest" } @@ -488,7 +484,7 @@ class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTr log.warn("readAttributes: Ignore ${qPath} for null bucket") } throw new UnsupportedOperationException("Not a valid Quilt Storage file attribute type: $type") - } + } @Override Map readAttributes(Path path, String attributes, LinkOption... options) throws IOException { diff --git a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF index dbeabc13..98790bf9 100644 --- a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF @@ -1,7 +1,7 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.quilt.QuiltPlugin Plugin-Id: nf-quilt -Plugin-Version: 0.8.6 +Plugin-Version: 0.8.7 Plugin-Provider: Quilt Data Plugin-Requires: >=22.10.6 diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy index f6154f6a..cfbb640a 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy @@ -14,15 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nextflow.quilt.nio +package nextflow.quilt -import nextflow.quilt.QuiltSpecification -import nextflow.quilt.QuiltObserver -import nextflow.quilt.jep.QuiltPackage +import nextflow.quilt.nio.QuiltPath +import nextflow.quilt.nio.QuiltPathFactory import nextflow.Session //import spock.lang.Ignore -import java.nio.file.Path import java.nio.file.Paths import groovy.transform.CompileDynamic @@ -33,15 +31,12 @@ import groovy.transform.CompileDynamic @CompileDynamic class QuiltObserverTest extends QuiltSpecification { - private static final String SPEC_KEY = 'udp-spec/nf-quilt/source' - private static final String TEST_KEY = 'bkt/pre/suf' - Session mockSession(boolean success = false) { String quilt_uri = 'quilt+s3://bucket#package=prefix%2fsuffix' return GroovyMock(Session) { getParams() >> [pubNot: 'foo', pubBad: 'foo:bar', outdir: SpecURI(), pubDir: testURI, inDir: quilt_uri] isSuccess() >> success - config >> [quilt: [outputPrefixes: ['pub']]] + config >> [quilt: [metadata: [key: 'value']]] workDir >> Paths.get('./work') } } @@ -51,126 +46,14 @@ class QuiltObserverTest extends QuiltSpecification { return observer } - void 'should extract appropriate UNIX Path asQuiltPath'() { - expect: - String unixFolder = "/var/tmp/output/${pkgString}" - Path unixPath = Paths.get(unixFolder) - QuiltObserver.asQuiltPath(unixPath).toString() == pkgString - where: - pkgString << ['quilt-example#package=examples%2fhurdat', 'udp-spec#package=nf-quilt%2fsource'] - } - - void 'should form pkgKey from QuiltPath'() { - given: - Path testPath = QuiltPathFactory.parse(testURI) - Path specPath = QuiltPathFactory.parse(SpecURI()) - expect: - QuiltObserver.pkgKey(testPath) == QuiltPackage.osConvert(TEST_KEY) - QuiltObserver.pkgKey(specPath) == QuiltPackage.osConvert(SPEC_KEY) - } - - void 'should extract quiltURIfromS3'() { - expect: - QuiltObserver.quiltURIfromS3(s3_uri) == quilt_uri - where: - s3_uri | quilt_uri - 's3://bucket/prefix/suffix' | 'quilt+s3://bucket#package=prefix%2fsuffix&dest=prefix%2fsuffix' - 's3://bucket/prefix' | 'quilt+s3://bucket#package=prefix%2fdefault_suffix&dest=prefix' - 's3://bucket' | 'quilt+s3://bucket#package=default_prefix%2fdefault_suffix&dest=/' - 's3://bucket/folder/prefix/suffix' | 'quilt+s3://bucket#package=prefix%2fsuffix&dest=folder%2fprefix%2fsuffix' - } - - void 'should return workRelative path for source'() { - given: - QuiltObserver observer = makeObserver() - Path workDir = observer.session.workDir - String subPath = 'output/file.txt' - String workPath = "job/hash/${subPath}" - Path source = Paths.get(workDir.toString(), workPath) - expect: - String relPath = observer.workRelative(source) - relPath == QuiltPackage.osConvert(subPath) - } - - void 'should return pkgRelative path for dest'() { - given: - QuiltObserver observer = makeObserver() - expect: - Path dest = Paths.get(TEST_KEY, folderPath) - String relPath = observer.pkgRelative(offset, dest) - rc == (relPath == QuiltPackage.osConvert(folderPath)) - where: - rc | offset | folderPath - true | TEST_KEY | 'output/file.txt' - false | SPEC_KEY | 'output/file.txt' - } - - void 'should findOutputParams'() { - given: - QuiltObserver observer = makeObserver() - String targetKey = QuiltPackage.osConvert('bucket/prefix/suffix') - expect: - String key = QuiltPackage.osConvert(unixKey) - observer.outputURIs - !observer.outputURIs.containsKey(targetKey) - observer.outputURIs.size() == 2 - - observer.outputURIs.containsKey(key) - observer.outputURIs[key] == uri - observer.confirmQuiltPath(QuiltPathFactory.parse(uri)) - where: - unixKey | uri - SPEC_KEY | SpecURI() - TEST_KEY | testURI - } - - void 'should set outputPrefixes from config'() { - given: - QuiltObserver observer = new QuiltObserver() - Map> config = ['quilt': ['outputPrefixes': ['bucket', 'file']]] - observer.checkConfig(config) - expect: - observer.outputPrefixes.size() == 2 - observer.outputPrefixes.contains('bucket') - observer.outputPrefixes.contains('file') - } - - void 'should not confirmQuiltPath for non-output URIs'() { + void 'should set metadata from config'() { given: QuiltObserver observer = new QuiltObserver() - QuiltPath specPath = QuiltPathFactory.parse(SpecURI()) - QuiltPath testPath = QuiltPathFactory.parse(testURI) + Map> config = [quilt: [metadata: [key: 'value']]] expect: - !observer.confirmQuiltPath(specPath) - !observer.confirmQuiltPath(testPath) - } - - void 'should return: #rc if canOverlayPath with: #path in: #root'() { - given: - QuiltObserver observer = makeObserver() - expect: - rc == observer.canOverlayPath(Paths.get(root, path), Paths.get(path)) - where: - rc | root | path - true | SPEC_KEY | 'output/file.txt' - true | TEST_KEY | 'output/file.txt' - false | '/root' | 'output/file.txt' - } - - /// source usually lacks subfolder paths - void 'should addOverlay logical path with subfolders'() { - given: - QuiltObserver observer = makeObserver() - String file_path = 'source' - Path source = Paths.get(root, file_path) - Path dest = Paths.get(root, path) - expect: - String relPath = observer.addOverlay(TEST_KEY, dest, source) - relPath == result ?: QuiltPackage.osConvert(result) - where: - root | path | result - TEST_KEY | SPEC_KEY | SPEC_KEY - SPEC_KEY | TEST_KEY | null + observer.configMetadata.size() == 0 + observer.checkConfig(config) + observer.configMetadata.size() == 1 } void 'should not error on onFlowComplete success'() { diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltPathifyTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltPathifyTest.groovy new file mode 100644 index 00000000..bf29d2e4 --- /dev/null +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltPathifyTest.groovy @@ -0,0 +1,126 @@ +/* groovylint-disable MethodName */ +/* + * Copyright 2022, Quilt Data Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nextflow.quilt + +import nextflow.quilt.jep.QuiltParser +import nextflow.quilt.nio.QuiltPath +import nextflow.quilt.nio.QuiltPathFactory + +// import java.nio.file.Path +// import java.nio.file.Paths +//import spock.lang.Ignore +import groovy.transform.CompileDynamic + +/** + * Test class for QuiltPathify + * + * Author: Ernest Prabhakar + */ + +@CompileDynamic +class QuiltPathifyTest extends QuiltSpecification { + + final static private String DP = 'default_prefix' + final static private String DS = 'default_suffix' + final static private String DB = QuiltParser.NULL_BUCKET + + QuiltPathify getPathify(String uri = SpecURI()) { + QuiltPath path = QuiltPathFactory.parse(uri) + QuiltPathify pathify = new QuiltPathify(path) + return pathify + } + + void 'test uriFromS3File'() { + expect: + def quilt_uri = QuiltPathify.uriFromS3File(s3path) + quilt_uri == expected + + where: + s3path | expected + '/bkt/pre/suf/fold/FILE.md' | 'quilt+s3://bkt#package=pre%2fsuf&path=fold/FILE.md' + '/bkt/pre/suf/FILE.md' | 'quilt+s3://bkt#package=pre%2fsuf&path=FILE.md' + '/bkt/pre/FILE.md' | "quilt+s3://bkt#package=pre%2f${DS}&path=FILE.md" + '/bkt/FILE.md' | "quilt+s3://bkt#package=${DP}%2f${DS}&path=FILE.md" + '/FILE.md' | "quilt+s3://${DB}#package=${DP}%2f${DS}&path=FILE.md" + } + + void 'test with QuiltPath'() { + when: + String uri = SpecURI() + QuiltPath path = QuiltPathFactory.parse(uri) + QuiltPathify pathify = new QuiltPathify(path) + + then: + pathify.isOverlay == false + pathify.uri == uri + pathify.path == path + getPathify().uri == uri + } + + void 'test findQuiltPath returns boolean'() { + when: + QuiltPathify pathify = getPathify() + + then: + rc == pathify.findQuiltPath(path) + + where: + rc | path + false | 'FILE.md' + true | 'bucket#package=prefix%2fsuffix&path=FILE.md' + } + + // Test findQuiltPath updates uri/path/pkg + void 'test findQuiltPath overrides attributes'() { + when: + QuiltPathify pathify = getPathify() + pathify.findQuiltPath('buck#package=prefix%2fsuffix&path=.%2fFILE.md') + + then: + pathify.isOverlay == false + pathify.uri == 'quilt+s3://buck#package=prefix%2fsuffix&path=.%2fFILE.md' + pathify.path.toString() == 'buck#package=prefix%2fsuffix&path=.%2fFILE.md' + pathify.pkg.toUriString() == 'quilt+s3://buck#package=prefix%2fsuffix&path=.%2fFILE.md' + } + + void 'test findQuiltPath preserves metadata'() { + when: + String now = QuiltProduct.now() + String meta = "meta=${now}" + String uriWith = uniqueQueryURI(meta) + String uriWithout = uriWith.replace("?$meta", '') + QuiltPathify pathify1 = getPathify(uriWith) + + then: + uriWith.contains(meta) + pathify1.uri == uriWith + pathify1.pkg.toUriString() == uriWith + + when: + QuiltPathify pathify2 = getPathify(uriWithout) + + then: + pathify2.pkgKey() == pathify1.pkgKey() + pathify2.uri == uriWithout + pathify2.pkg.toUriString() == uriWith + } + + // Test makeQuiltPath creates new uri/path/pkg + // Test makeQuiltPath sets isOverlay + // Test copyToPackage copies overly file to package folder + +} diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltPkgTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltPkgTest.groovy index 668df423..017c4282 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltPkgTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltPkgTest.groovy @@ -43,9 +43,7 @@ class QuiltPkgTest extends QuiltSpecification { String baseURI = SpecURI().replace('source', suffix) QuiltPathFactory factory = new QuiltPathFactory() QuiltPath qpath = factory.parseUri(baseURI) - println("Parsed: $qpath") QuiltPackage pkg = qpath.pkg() - println("Package: $pkg") return pkg } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy index 727c0674..a5fa21b3 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nextflow.quilt.nio +package nextflow.quilt import nextflow.Session import nextflow.script.WorkflowMetadata -import nextflow.quilt.QuiltSpecification -import nextflow.quilt.QuiltProduct +import nextflow.quilt.nio.QuiltPath +import nextflow.quilt.nio.QuiltPathFactory import nextflow.quilt.jep.QuiltParser import nextflow.quilt.jep.QuiltPackage @@ -39,29 +39,36 @@ import spock.lang.Unroll @CompileDynamic class QuiltProductTest extends QuiltSpecification { - QuiltProduct makeProduct(String query=null, boolean success = false) { - String subURL = query ? testURI.replace('key=val&key2=val2', query) : testURI - WorkflowMetadata metadata = GroovyMock(WorkflowMetadata) { + QuiltProduct makeProductFromUrl(String url, boolean success = false) { + WorkflowMetadata wf_meta = GroovyMock(WorkflowMetadata) { toMap() >> [start:'2022-01-01', complete:'2022-01-02'] } + QuiltPath path = QuiltPathFactory.parse(url) + QuiltPathify pathify = new QuiltPathify(path) Session session = GroovyMock(Session) { - getWorkflowMetadata() >> metadata - getParams() >> [outdir: subURL] + getWorkflowMetadata() >> wf_meta + getParams() >> [outdir: url] isSuccess() >> success + config >> [quilt: [metadata: [cfkey: 'cfval']], runName: 'my-run', publishing: false] + } + return new QuiltProduct(pathify, session) + } + + QuiltProduct makeProduct(String query=null, boolean success = false) { + if (query == null) { + return makeProductFromUrl(testURI, success) } - QuiltPath path = QuiltPathFactory.parse(subURL) - return new QuiltProduct(path, session) + String subURL = uniqueQueryURI(query) + return makeProductFromUrl(subURL, success) } QuiltProduct makeWriteProduct(Map meta = [:]) { - String subURL = writeableURL('quilt_product_test') + '&workflow=my-workflow' + String subURL = writeableURI('quilt_product_test') // + '&workflow=universal' if (meta) { String query = QuiltParser.unparseQuery(meta) subURL = subURL.replace('#', "?${query}#") } - Session session = GroovyMock(Session) - QuiltPath path = QuiltPathFactory.parse(subURL) - return new QuiltProduct(path, session) + return makeProductFromUrl(subURL) } void 'should generate mocks from makeProduct'() { @@ -73,6 +80,9 @@ class QuiltProductTest extends QuiltSpecification { product.pkg product.session != null product.session.getWorkflowMetadata() != null + product.meta != null + product.meta.size() == 4 + product.meta.key == 'val' } void 'should generate solid string for timestamp from now'() { @@ -80,7 +90,7 @@ class QuiltProductTest extends QuiltSpecification { def now = QuiltProduct.now() then: now - now.contains('T') + now.contains('t') !now.contains(' ') } @@ -98,11 +108,11 @@ class QuiltProductTest extends QuiltSpecification { void 'shouldSkip is true if key=SKIP'() { given: - QuiltProduct product = makeProduct('readme=SKIP&metadata=SKIP') + QuiltProduct product = makeProduct('readme=SKIP') expect: !product.shouldSkip(QuiltProduct.KEY_SKIP) + !product.shouldSkip(QuiltProduct.KEY_META) product.shouldSkip(QuiltProduct.KEY_README) - product.shouldSkip(QuiltProduct.KEY_META) !makeProduct('?readme=now').shouldSkip() } @@ -119,12 +129,21 @@ class QuiltProductTest extends QuiltSpecification { } void 'always creates README if readme!=SKIP'() { - given: + when: + QuiltProduct defaultREADME = makeProduct() + String text = defaultREADME.setupReadme() + def files = defaultREADME.pkg.folder.list().sort() + + then: + !defaultREADME.shouldSkip(QuiltProduct.KEY_README) + files.size() == 1 + + when: String readme_text = 'hasREADME' QuiltProduct hasREADME = makeProduct("readme=${readme_text}") - String text = hasREADME.setupReadme() - def files = hasREADME.pkg.folder.list().sort() - expect: + text = hasREADME.setupReadme() + files = hasREADME.pkg.folder.list().sort() + then: text == readme_text !hasREADME.shouldSkip(QuiltProduct.KEY_README) files.size() == 1 @@ -150,6 +169,23 @@ class QuiltProductTest extends QuiltSpecification { quilt_summarize.size() == 1 } + @IgnoreIf({ env.WRITE_BUCKET == null }) + void 'should copyFile'() { + given: + QuiltProduct product = makeWriteProduct() + String filename = 'test.md' + String text = 'test' + Path src = Paths.get(product.pkg.folder.toString(), filename) + Path dest = Paths.get(product.pkg.folder.toString(), 'copy', filename) + Files.writeString(src, text) + + when: + product.copyFile(src, dest.toString(), text) + + then: + Files.exists(dest) + } + @Ignore('Not implemented yet: pushes previous metadata') void 'pushes previous metadata if metadata=SKIP'() { given: @@ -199,7 +235,7 @@ class QuiltProductTest extends QuiltSpecification { Path outPath = Paths.get(root, filename) outPath.getParent().toFile().mkdirs() Files.writeString(outPath, "#Time, Filename\n${timestamp},${filename}") - // println("writeFile: ${filename} -> ${outPath}") + // println("writeFile: ${filename} -> ${outPath}") } int writeFiles(dest) { @@ -222,7 +258,7 @@ class QuiltProductTest extends QuiltSpecification { @Ignore('Invalid test: top-level summarize') void 'should summarize top-level readable files + multiqc '() { given: - String sumURL = writeableURL('summarized') + String sumURL = writeableURI('summarized') QuiltPackage sumPkg = writeablePackage('summarized') writeFiles(sumPkg.packageDest()) @@ -250,15 +286,27 @@ class QuiltProductTest extends QuiltSpecification { expect: quilt_meta != null + quilt_meta.config == meta } - void 'should setupMeta from session'() { - given: - QuiltProduct product = makeProduct() - Map quilt_meta = product.setupMeta() + void 'should addSessionMeta from session'() { + when: + QuiltProduct product = makeProduct('a=b&c=d') + Map start_meta = product.meta - expect: - quilt_meta != null + then: + start_meta != null + start_meta.size() == 4 + start_meta.a == 'b' + product.addSessionMeta() + + when: + Map end_meta = product.meta + + then: + end_meta != null + end_meta.size() > 4 + end_meta.a == 'b' } void 'should throw error on publish'() { diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy index b38ab544..0f8f3aea 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy @@ -48,7 +48,7 @@ import spock.lang.Specification class QuiltSpecification extends Specification { static String SpecURI() { - return 'quilt+s3://udp-spec#package=nf-quilt/source' + return 'quilt+s3://udp-spec#package=nf-quilt%2fsource' } @Shared String testURI @@ -92,13 +92,17 @@ class QuiltSpecification extends Specification { Plugins.startIfMissing('nf-quilt') } - String writeableURL(String suffix='QuiltSpecification') { + String writeableURI(String suffix='QuiltSpecification') { return "quilt+s3://${writeBucket}#package=test/${suffix}&force=true" } + String uniqueQueryURI(String query = 'key=val') { + return "quilt+s3://${QuiltProduct.now()}?${query}#package=test%2funique" + } + QuiltPackage writeablePackage(String suffix, String workflow=null) { QuiltPathFactory factory = new QuiltPathFactory() - String url = writeableURL(suffix) + String url = writeableURI(suffix) if (workflow) { url += "&workflow=${workflow}" } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy index 9f8e93d4..d84bc6ef 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy @@ -125,7 +125,6 @@ class QuiltPackageTest extends QuiltSpecification { def qpkg = qpath.pkg() Path outputFolder = pkg.packageDest() Path readmeFile = outputFolder.resolve('README.md') - println("qpkg: ${qpkg} -> ${qpath.localPath()} == ${readmeFile}") expect: !qpath.isJustPackage() @@ -193,18 +192,14 @@ class QuiltPackageTest extends QuiltSpecification { void 'should fail pushing new files to read-only bucket '() { given: - println("read-only-bucket:TEST_URL: ${READONLY_URL}") def qout = factory.parseUri(READONLY_URL) def opkg = qout.pkg() opkg.install() - println("opkg: ${opkg} installed: ${opkg.isInstalled()}") def outPath = Paths.get(opkg.packageDest().toString(), 'foo/bar.txt') - println("outPath: ${outPath}") Files.writeString(outPath, "Time: ${timestamp}") expect: Files.exists(outPath) when: - println('Pushing to read-only bucket') opkg.push() then: thrown(RuntimeException) diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltParserTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltParserTest.groovy index dece83a1..2b97167b 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltParserTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltParserTest.groovy @@ -127,7 +127,6 @@ class QuiltParserTest extends QuiltSpecification { when: String query = 'key=val1,val2&quay=vale1&quay=vale2' Map result = QuiltParser.parseQuery(query) - println "QuiltParserTest[$query] -> ${result}" String unparsed = QuiltParser.unparseQuery(result) then: diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy index a4a71f6c..b302091f 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy @@ -111,7 +111,7 @@ class QuiltFileSystemProviderTest extends QuiltSpecification { void 'should upload file to test bucket'() { given: QuiltFileSystemProvider provider = new QuiltFileSystemProvider() - String url = writeableURL('upload') + String url = writeableURI('upload') String filename = 'UPLOAD_THIS.md' QuiltPath remotePath = QuiltPathFactory.parse(url) QuiltPath remoteFile = remotePath.resolveSibling(filename) @@ -124,7 +124,8 @@ class QuiltFileSystemProviderTest extends QuiltSpecification { !Files.exists(remoteFile.localPath()) when: - provider.upload(tempFile, remoteFile) + CopyOption opt = StandardCopyOption.REPLACE_EXISTING + provider.upload(tempFile, remoteFile, opt) then: Files.exists(remoteFile) diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy index 59ff61d3..f24b808a 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy @@ -307,7 +307,6 @@ class QuiltNioTest extends QuiltSpecification { void 'should create temp file and directory'() { given: Path base = Paths.get(new URI(PACKAGE_URL)).toAbsolutePath() - println "BASE: ${base}" when: Path t1 = Files.createTempDirectory(base, 'test') diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy index 4f24cb44..6360f068 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy @@ -339,7 +339,7 @@ class QuiltPathTest extends QuiltSpecification { expect: subPath subPath.toString() == 'bucket#package=so%2fme&path=name.txt' - } + } void 'should match endsWith'() { given: diff --git a/settings.gradle b/settings.gradle index 586dc7ab..0dc7ad64 100644 --- a/settings.gradle +++ b/settings.gradle @@ -15,7 +15,7 @@ */ plugins { - id("org.gradle.toolchains.foojay-resolver-convention") version "0.8.0" + id('org.gradle.toolchains.foojay-resolver-convention') version '0.8.0' } rootProject.name = 'nf-quilt'