Skip to content

Commit

Permalink
feat(244) dynamic uris (#248)
Browse files Browse the repository at this point in the history
#244

---------

Co-authored-by: Dr. Ernie Prabhakar <[email protected]>
  • Loading branch information
drernie and drernie authored Oct 24, 2024
1 parent 9c64212 commit 847d29f
Show file tree
Hide file tree
Showing 29 changed files with 608 additions and 505 deletions.
6 changes: 4 additions & 2 deletions .github/workflows/mega-linter.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,16 @@ on:
# Trigger mega-linter at every push. Action will also be visible from Pull Requests to main
push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions)

permissions: read-all
permissions:
issues: write
pull-requests: write

env: # Comment env block if you do not want to apply fixes
# Apply linter fixes configuration
APPLY_FIXES: all # When active, APPLY_FIXES must also be defined as environment variable (in github/workflows/mega-linter.yml or other CI tool)
#APPLY_FIXES_EVENT: pull_request # Decide which event triggers application of fixes in a commit or a PR (pull_request, push, all)
#APPLY_FIXES_MODE: pull_request # If APPLY_FIXES is used, defines if the fixes are directly committed (commit) or posted in a PR (pull_request)
DISABLE_LINTERS: SPELL_CSPELL,COPYPASTE_JSCPD,REPOSITORY_GITLEAKS
DISABLE_LINTERS: SPELL_CSPELL,COPYPASTE_JSCPD,REPOSITORY_GITLEAKS,GROOVY_NPM_GROOVY_LINT
FILTER_REGEX_EXCLUDE: .*/.*gradle

concurrency:
Expand Down
3 changes: 3 additions & 0 deletions .groovylintrc.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@
"JUnitPublicNonTestMethod": {
"enabled": false
},
"JUnitTestMethodWithoutAssert": {
"enabled": false
},
"JavaIoPackageAccess": {
"enabled": false
},
Expand Down
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
# Changelog

## [0.8.7] 2024-10-23

- Use package cache instead of `params` to find output URIs
(in order to support dynamic URIs set by, e.g. `main.nf`)
- Allow setting metadata from inside the workflow

## [0.8.6] 2024-09-11

- Fix addOverlay bug on subfolders
Expand Down
8 changes: 4 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ REPORT ?= ./plugins/$(PROJECT)/build/reports/tests/test/index.html

verify: #compile
echo $(WRITE_BUCKET)
./gradlew check || open $(REPORT)
./gradlew test ${ONE} || open $(REPORT)

fast:
./gradlew ${mm}test --fail-fast || open ./plugins/nf-quilt/build/reports/tests/test/index.html

./gradlew test ${ONE} --fail-fast || open $(REPORT)
# example: make fast ONE="--tests QuiltProductTest"
check-env:
echo $(VERSION)
echo $(WRITE_BUCKET)
Expand All @@ -38,7 +38,7 @@ compile:
./gradlew compileGroovy exportClasspath
@echo "DONE `date`"

nextflow-git:
nextflow:
if [ ! -d "$(NF_DIR)" ]; then git clone https://github.com/nextflow-io/nextflow.git "$(NF_DIR)"; fi
cd "$(NF_DIR)"; git checkout && make compile && git restore .; cd ..

Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,8 @@ From the command-line, do, e.g.:
```bash
# export NXF_VER=23.04.3
export LOG4J_DEBUG=true # for verbose logging
export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.8.6/nf-quilt-0.8.6-meta.json
nextflow run main.nf -plugins [email protected].6
export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.8.7/nf-quilt-0.8.7-meta.json
nextflow run main.nf -plugins [email protected].7
```

For Tower, you can use the "Pre-run script" to set the environment variables.
Expand Down
3 changes: 1 addition & 2 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -194,9 +194,8 @@ def getRuntimeConfigs() {
task exportClasspath {
dependsOn allprojects.jar
doLast {
def home = System.getProperty('user.home')
def all = getRuntimeConfigs()
def libs = all.collect { File file -> /*println file.canonicalPath.replace(home, '$HOME');*/ file.canonicalPath; }
def libs = all.collect { File file -> file.canonicalPath; }
['nextflow', 'nf-commons', 'nf-httfs'].each { libs << file("modules/$it/build/libs/${it}-${version}.jar").canonicalPath }
file('.launch.classpath').text = libs.unique().join(':')
}
Expand Down
98 changes: 49 additions & 49 deletions gradle-groovysh-init.gradle
Original file line number Diff line number Diff line change
@@ -1,49 +1,49 @@
gradle.projectsLoaded {
rootProject {
afterEvaluate { project ->
if (!project.repositories.any{it.name == 'MavenRepo'}) {
project.repositories {
// To be able to load org.codehaus.groovy:groovy-groovysh
mavenCentral()
}
}

project.configurations {
groovyshdependencies
}

project.dependencies {
groovyshdependencies("org.codehaus.groovy:groovy-groovysh:${GroovySystem.version}") {
exclude group: 'org.codehaus.groovy'
}
}

project.tasks.register('groovysh') {
group 'debug'
description 'Runs an interactive shell in the context of the project.'
doLast {
URLClassLoader groovyObjectClassLoader = GroovyObject.class.classLoader
def groovyshClass
def groovyShell

// Add dependency jars to classloader
configurations.groovyshdependencies.each {File file ->
groovyObjectClassLoader.addURL(file.toURL())
}
Class.forName('jline.console.history.FileHistory', true, groovyObjectClassLoader)
groovyshClass = Class.forName('org.codehaus.groovy.tools.shell.Groovysh', true, groovyObjectClassLoader)

if (groovyshClass) {
groovyShell = groovyshClass.newInstance()
}
if (groovyShell) {
groovyShell.interp.context.variables.put("gradle", gradle)
groovyShell.interp.context.variables.put("settings", gradle.settings)
groovyShell.interp.context.variables.put("project", project)
groovyShell.run('')
}
}
}
}
}
}
gradle.projectsLoaded {
rootProject {
afterEvaluate { project ->
if (!project.repositories.any { it.name == 'MavenRepo' }) {
project.repositories {
// To be able to load org.codehaus.groovy:groovy-groovysh
mavenCentral()
}
}

project.configurations {
groovyshdependencies
}

project.dependencies {
groovyshdependencies("org.codehaus.groovy:groovy-groovysh:${GroovySystem.version}") {
exclude group: 'org.codehaus.groovy'
}
}

project.tasks.register('groovysh') {
group 'debug'
description 'Runs an interactive shell in the context of the project.'
doLast {
URLClassLoader groovyObjectClassLoader = GroovyObject.classLoader
def groovyshClass
def groovyShell

// Add dependency jars to classloader
configurations.groovyshdependencies.each { File file ->
groovyObjectClassLoader.addURL(file.toURL())
}
Class.forName('jline.console.history.FileHistory', true, groovyObjectClassLoader)
groovyshClass = Class.forName('org.codehaus.groovy.tools.shell.Groovysh', true, groovyObjectClassLoader)

if (groovyshClass) {
groovyShell = groovyshClass.newInstance()
}
if (groovyShell) {
groovyShell.interp.context.variables.put('gradle', gradle)
groovyShell.interp.context.variables.put('settings', gradle.settings)
groovyShell.interp.context.variables.put('project', project)
groovyShell.run('')
}
}
}
}
}
}
106 changes: 52 additions & 54 deletions groovysh-task.gradle
Original file line number Diff line number Diff line change
@@ -1,54 +1,52 @@
gradle.projectsLoaded {
rootProject {
afterEvaluate { project ->
if (!project.repositories.any{it.name == 'MavenRepo'}) {
project.repositories {
// To be able to load org.apache.groovy:groovy-groovysh and dependencies
mavenCentral {
content {
includeGroup 'org.apache.groovy'
includeGroup 'jline'
includeGroup 'com.github.javaparser'
includeGroup 'org.ow2.asm'
includeGroup 'org.abego.treelayout'
includeGroup 'org.apache.ivy'
}
}
}
}
project.configurations {
groovyshdependencies
}

project.dependencies {
groovyshdependencies "org.apache.groovy:groovy-groovysh:4.0.23"
}

project.tasks.register('groovysh') {
group 'debug'
description 'Runs an interactive shell in the context of the project. Use :inspect command to inspect project, gradle, settings or other objects.'
doLast {
URLClassLoader groovyshClassLoader = new URLClassLoader();
configurations.groovyshdependencies.each {File file ->
groovyshClassLoader.addURL(file.toURI().toURL())
}

def fileHistoryClass
def groovyshClass
def groovyShell
fileHistoryClass = Class.forName('jline.console.history.FileHistory', true, groovyshClassLoader)
groovyshClass = Class.forName('org.apache.groovy.groovysh.Groovysh', true, groovyshClassLoader)
if (groovyshClass) {
groovyShell = groovyshClass.newInstance()
if (groovyShell) {
groovyShell.interp.context.variables.put("gradle", gradle)
groovyShell.interp.context.variables.put("settings", gradle.settings)
groovyShell.interp.context.variables.put("project", project)
groovyShell.run('# Available objects: gradle, settings, project\n# Try :inspect project')
}
}
}
}
}
}
}
gradle.projectsLoaded {
rootProject {
afterEvaluate { project ->
if (!project.repositories.any { it.name == 'MavenRepo' }) {
project.repositories {
// To be able to load org.apache.groovy:groovy-groovysh and dependencies
mavenCentral {
content {
includeGroup 'org.apache.groovy'
includeGroup 'jline'
includeGroup 'com.github.javaparser'
includeGroup 'org.ow2.asm'
includeGroup 'org.abego.treelayout'
includeGroup 'org.apache.ivy'
}
}
}
}
project.configurations {
groovyshdependencies
}

project.dependencies {
groovyshdependencies 'org.apache.groovy:groovy-groovysh:4.0.23'
}

project.tasks.register('groovysh') {
group 'debug'
description 'Runs an interactive shell in the context of the project. Use :inspect command to inspect project, gradle, settings or other objects.'
doLast {
URLClassLoader groovyshClassLoader = new URLClassLoader()
configurations.groovyshdependencies.each { File file ->
groovyshClassLoader.addURL(file.toURI().toURL())
}

def groovyshClass
def groovyShell
groovyshClass = Class.forName('org.apache.groovy.groovysh.Groovysh', true, groovyshClassLoader)
if (groovyshClass) {
groovyShell = groovyshClass.newInstance()
if (groovyShell) {
groovyShell.interp.context.variables.put('gradle', gradle)
groovyShell.interp.context.variables.put('settings', gradle.settings)
groovyShell.interp.context.variables.put('project', project)
groovyShell.run('# Available objects: gradle, settings, project\n# Try :inspect project')
}
}
}
}
}
}
}
26 changes: 13 additions & 13 deletions plugins/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@
*/

plugins {
id "java"
id "groovy"
id "io.nextflow.nf-build-plugin" version "1.0.1"
id 'java'
id 'groovy'
id 'io.nextflow.nf-build-plugin' version '1.0.1'
}

ext.github_organization = project.findProperty('github_organization') ?: 'nextflow-io'
Expand All @@ -29,8 +29,9 @@ ext.github_index_url = "https://github.com/${github_organization}/plugins/main/p
jar.enabled = false

String computeSha512(File file) {
if( !file.exists() )
if (!file.exists()) {
throw new GradleException("Missing file: $file -- cannot compute SHA-512")
}
return org.apache.commons.codec.digest.DigestUtils.sha512Hex(file.bytes)
}

Expand All @@ -40,15 +41,15 @@ String now() {

List<String> allPlugins() {
def plugins = []
new File(rootProject.rootDir, 'plugins') .eachDir { if(it.name.startsWith('nf-')) plugins.add(it.name) }
new File(rootProject.rootDir, 'plugins') .eachDir { if (it.name.startsWith('nf-')) plugins.add(it.name) }
return plugins
}

String metaFromManifest(String meta, File file) {
def str = file.text
def regex = ~/(?m)^$meta:\s*([\w-\.<>=]+)$/
def m = regex.matcher(str)
if( m.find() ) {
if (m.find()) {
def ver = m.group(1)
println "Set plugin '${file.parentFile.parentFile.parentFile.parentFile.name}' version=${ver}"
return ver
Expand All @@ -68,7 +69,7 @@ subprojects {
mavenCentral()
}

version = metaFromManifest('Plugin-Version',file('src/resources/META-INF/MANIFEST.MF'))
version = metaFromManifest('Plugin-Version', file('src/resources/META-INF/MANIFEST.MF'))

tasks.withType(Jar) {
duplicatesStrategy = DuplicatesStrategy.INCLUDE
Expand All @@ -94,7 +95,7 @@ subprojects {
"version": "${project.version}",
"date": "${timestamp}",
"url": "https://github.com/${github_organization}/${project.name}/releases/download/${project.version}/${project.name}-${project.version}.zip",
"requires": "${metaFromManifest('Plugin-Requires',file('src/resources/META-INF/MANIFEST.MF'))}",
"requires": "${metaFromManifest('Plugin-Requires', file('src/resources/META-INF/MANIFEST.MF'))}",
"sha512sum": "${computeSha512(zip)}"
}
"""
Expand Down Expand Up @@ -129,10 +130,10 @@ subprojects {
/*
* "install" the plugin the project root build/plugins directory
*/
project.parent.tasks.getByName("assemble").dependsOn << copyPluginZip
project.parent.tasks.getByName('assemble').dependsOn << copyPluginZip

task uploadPlugin(type: io.nextflow.gradle.tasks.GithubUploader, dependsOn: makeZip) {
assets = providers.provider {["$buildDir/libs/${project.name}-${project.version}.zip",
assets = providers.provider { ["$buildDir/libs/${project.name }-${project.version }.zip",
"$buildDir/libs/${project.name}-${project.version}-meta.json" ]}
release = providers.provider { project.version }
repo = providers.provider { project.name }
Expand All @@ -141,7 +142,6 @@ subprojects {
authToken = github_access_token
skipExisting = true
}

}

task upload(dependsOn: [subprojects.uploadPlugin]) { }
Expand All @@ -151,11 +151,11 @@ classes.dependsOn subprojects.copyPluginLibs
/*
* Merge and publish the plugins index file
*/
task publishIndex( type: io.nextflow.gradle.tasks.GithubRepositoryPublisher ) {
task publishIndex(type: io.nextflow.gradle.tasks.GithubRepositoryPublisher) {
indexUrl = github_index_url
repos = allPlugins()
owner = github_organization
githubUser = github_username
githubEmail = github_commit_email
githubToken = github_access_token
}
}
Loading

0 comments on commit 847d29f

Please sign in to comment.