diff --git a/.github/workflows/mega-linter.yml b/.github/workflows/mega-linter.yml index 2a10ed4d..a58224a7 100644 --- a/.github/workflows/mega-linter.yml +++ b/.github/workflows/mega-linter.yml @@ -27,9 +27,9 @@ jobs: steps: # Git Checkout - name: Checkout Code - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 with: - token: ${{ secrets.GITHUB_TOKEN }} # secrets.PAT || + token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} fetch-depth: 0 # If you use VALIDATE_ALL_CODEBASE = true, you can remove this line to improve performances # MegaLinter @@ -37,7 +37,7 @@ jobs: id: ml # You can override MegaLinter flavor used to have faster performances # More info at https://megalinter.github.io/flavors/ - uses: oxsecurity/megalinter/flavors/dotnet@v7.4.0 + uses: oxsecurity/megalinter/flavors/dotnet@v7.3.0 env: # All available variables are described in documentation # https://megalinter.github.io/configuration/ @@ -48,6 +48,7 @@ jobs: # Upload MegaLinter artifacts - name: Archive production artifacts + if: ${{ success() }} || ${{ failure() }} uses: actions/upload-artifact@v3 with: name: MegaLinter reports diff --git a/.github/workflows/pkg-test.yml b/.github/workflows/pkg-test.yml new file mode 100644 index 00000000..51317a7c --- /dev/null +++ b/.github/workflows/pkg-test.yml @@ -0,0 +1,61 @@ +name: Test + +on: + # Trigger at every push. Action will also be visible from Pull Requests to master + push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions) + pull_request: + branches: [master] + +permissions: read-all + +jobs: + build: + name: Package Test + permissions: + contents: read + id-token: write + issues: write + pull-requests: write + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + java_version: [19] + runs-on: ${{ matrix.os }} + + steps: + # Git Checkout + - name: Checkout Code + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: arn:aws:iam::712023778557:role/GitHub-Testing-NF-Quilt + aws-region: us-east-1 + + - name: Setup Java ${{matrix.java_version}} + uses: actions/setup-java@v3 + with: + java-version: ${{matrix.java_version}} + distribution: 'temurin' + architecture: x64 + cache: gradle + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + + - name: Run Package Tests + run: make pkg-test + + - name: Archive production artifacts + if: ${{ success() }} || ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: nf-quilt-pkg-test + path: | + /home/runner/work/nf-quilt/nf-quilt/plugins/nf-quilt/build/reports/tests/test/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 809d703d..4f3a22e4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,8 +12,8 @@ jobs: build: name: Test permissions: - # contents: read - # id-token: write + contents: read + id-token: write issues: write pull-requests: write @@ -27,17 +27,16 @@ jobs: steps: # Git Checkout - name: Checkout Code - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 with: token: ${{ secrets.GITHUB_TOKEN }} fetch-depth: 0 - - name: Install Quilt3 - uses: actions/setup-python@v4 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v3 with: - python-version: '3.10' - - run: python -m pip install quilt3 - - run: which quilt3 + role-to-assume: arn:aws:iam::712023778557:role/GitHub-Testing-NF-Quilt + aws-region: us-east-1 - name: Setup Java ${{matrix.java_version}} uses: actions/setup-java@v3 @@ -54,6 +53,7 @@ jobs: run: make test - name: Archive production artifacts + if: ${{ success() }} || ${{ failure() }} uses: actions/upload-artifact@v3 with: name: nf-quilt-test-reports diff --git a/CHANGELOG.md b/CHANGELOG.md index 713cf674..36d097b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,24 @@ # Changelog +## [0.7.0] 2023-10-05 + +- Officially QuiltCore 0.1.0 instead of Python + ## [0.6.0] 2023-10-03 +(interim release still using Python) + - Stop overwriting README.md (use README_NF_QUILT.md instead) - improve formatting - Smarter quilt_summarize.json - Top level (only): md, html, pdf, csv, tsv - multiqc sub-folder HTML +## [0.5.0] 2023-09-04 + +- Switch to quiltcore-java [NOTE: this pre-release does NOT check workflows] +- Do not pre-install packages (only install before write) + ## [0.4.5] 2023-08-23 - fix metadata in README.md diff --git a/Makefile b/Makefile index 134ac862..471ac934 100644 --- a/Makefile +++ b/Makefile @@ -4,13 +4,10 @@ WRITE_BUCKET ?= quilt-example FRAGMENT ?= &path=. NF_DIR ?= ../nextflow PID ?= $$$$ -PIP ?= python -m pip PIPELINE ?= sarek QUERY ?= ?Name=$(USER)&Owner=Kevin+Moore&Date=2023-03-07&Type=CRISPR&Notebook+URL=http%3A%2F%2Fexample.com TEST_URI ?= quilt+s3://$(WRITE_BUCKET)$(QUERY)\#package=test/hurdat$(FRAGMENT) QUILT_URI ?= quilt+s3://$(WRITE_BUCKET)\#package=$(PROJECT)/$(PIPELINE) -PIP ?= pip3 -QUILT3 ?= /usr/local/bin/quilt3 REPORT ?= ./plugins/$(PROJECT)/build/reports/tests/test/index.html verify: #compile @@ -35,13 +32,9 @@ compile: nextflow: if [ ! -d "$(NF_DIR)" ]; then git clone https://github.com/nextflow-io/nextflow.git "$(NF_DIR)"; fi - pushd "$(NF_DIR)"; git checkout && make compile && git restore .; popd + cd "$(NF_DIR)"; git checkout && make compile && git restore .; cd .. -install-python: - if [ ! -x "$(QUILT3)" ]; then $(PIP) install quilt3 ; fi - which quilt3 - -compile-all: install-python nextflow compile +compile-all: nextflow compile check: ./gradlew check --warning-mode all diff --git a/README.md b/README.md index 5892a478..6c173b02 100644 --- a/README.md +++ b/README.md @@ -6,33 +6,15 @@ NextFlow plugin for reading and writing Quilt packages as a FileSystem developed by [Quilt Data](https://quiltdata.com/) that enables you read and write directly to Quilt packages using `quilt+s3` URIs wherever your NextFlow pipeline currently use `s3` URIs. -Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs +Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs. ## I. Using the nf-quilt plugin in Production This plugin allows your existing pipelines, without modification, to read and write versioned Quilt packages stored on Amazon S3. -Use the following four steps to configure NextFlow Tower or your command-line environment. - -1. Install the `quilt3` command-line tool - -This is distributed as an open source Python package you can install using `pip3`, -and must be available in the PATH used by `nextflow`. - -```bash -yum install python3-pip -y -yum install git -y -pip3 install quilt3 -which quilt3 -``` - -The above instructions use the 'yum' package manager, -which NextFlow Tower uses in the "Pre-run script" -when you edit the Pipeline settings from the Launchpad. - -If you are running from the command-line, you may need to use your own package manager -(or just skip those lines if you already have Python and Git). +Use the following three steps to configure NextFlow Tower or your command-line environment. +[Note: versions 0.7.0 and later no longer require the `quilt3` Python client.] 1. Enable the `nf-quilt` plugin @@ -93,8 +75,8 @@ From the command-line, do, e.g.: ```bash # export NXF_VER=23.04.3 -export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.6.0/nf-quilt-0.6.0-meta.json -nextflow run main.nf -plugins nf-quilt@0.6.0 +export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.7.0/nf-quilt-0.7.0-meta.json +nextflow run main.nf -plugins nf-quilt@0.7.0 ``` For Tower, you can use the "Pre-run script" to set the environment variables. @@ -183,13 +165,6 @@ git clone https://github.com/quiltdata/nf-quilt.git cd ./nf-quilt ``` -You also need to use Python to install the `quilt3` command-line tool used by `nf-quilt`: - -```bash -pip install quilt3 -which quilt3 -``` - ### Unit Testing You can compile run unit tests with: diff --git a/plugins/build.gradle b/plugins/build.gradle index 1e067cdd..ed48d2c5 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -108,6 +108,7 @@ subprojects { task copyPluginLibs(type: Sync) { from configurations.runtimeClasspath into 'build/target/libs' + duplicatesStrategy 'exclude' } /* diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index e7c747d0..c40f1501 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -59,6 +59,9 @@ ext{ } dependencies { + // quiltcore + implementation 'com.quiltdata:quiltcore:0.1' + // This dependency is exported to consumers, that is to say found on their compile classpath. compileOnly "io.nextflow:nextflow:$nextflowVersion" compileOnly 'org.slf4j:slf4j-api:2.0.9' diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index 797d567a..57daef2a 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -28,6 +28,17 @@ import java.nio.file.attribute.BasicFileAttributes import java.util.stream.Collectors import java.time.LocalDate +import com.quiltdata.quiltcore.Entry +import com.quiltdata.quiltcore.Registry +import com.quiltdata.quiltcore.Namespace +import com.quiltdata.quiltcore.Manifest +import com.quiltdata.quiltcore.key.LocalPhysicalKey +import com.quiltdata.quiltcore.key.S3PhysicalKey +import com.quiltdata.quiltcore.workflows.WorkflowException + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + @Slf4j @CompileStatic class QuiltPackage { @@ -43,7 +54,6 @@ class QuiltPackage { private final Path folder private final Map meta private boolean installed - private String cmd = 'N/A' static String today() { LocalDate date = LocalDate.now() @@ -86,7 +96,7 @@ class QuiltPackage { log.debug("${pkg}: attempting install for.pkgKey $pkgKey (okay if fails)") pkg.install() } - catch (Exception e) { + catch (IOException e) { log.warn("Package `${parsed.toUriString()}` does not yet exist") } return pkg @@ -166,51 +176,10 @@ class QuiltPackage { this.installed = false } - String key_dest() { - return "--dest ${packageDest()}" - } - - String key_dir() { - return "--dir ${packageDest()}" - } - boolean is_force() { return parsed.options[QuiltParser.P_FORCE] } - String key_force() { - //log.debug("key_force.options[${parsed.options[QuiltParser.P_FORCE]}] ${parsed.options}") - return is_force() ? '--force' : '' - } - - String key_hash() { - return (hash == 'latest' || hash == null || hash == 'null') ? '' : "--top-hash $hash" - } - - String key_meta(Map srcMeta = [:]) { - //log.debug("key_meta.srcMeta $srcMeta") - //log.debug("key_meta.uriMeta ${meta}") - Map metas = srcMeta + meta - if (metas.isEmpty()) { return '' } - - String jsonMeta = toJson(metas) - log.debug("key_meta.jsonMeta $jsonMeta") - return "--meta '$jsonMeta'" - } - - String key_msg(String message='') { - String msg = meta_overrides('msg', "nf-quilt:${today()}-${message}") - return "--message '${sanitize(msg)}'" - } - - String key_registry() { - return "--registry s3://${bucket}" - } - - String key_workflow() { - return (parsed.workflowName) ? "--workflow ${parsed.workflowName}" : '' - } - boolean isInstalled() { return installed } @@ -219,48 +188,29 @@ class QuiltPackage { return folder } - String lastCommand() { - return cmd - } - - int call(String... args) { - def command = ['quilt3'] - command.addAll(args) - cmd = command.join(' ') - log.debug("QuiltPackage.call `${cmd}`") + Path install() { + Path dest = packageDest() try { - ProcessBuilder pb = new ProcessBuilder('bash', '-c', cmd) - pb.redirectErrorStream(true) - - Process p = pb.start() - //log.debug("call.start ${p}") - String result = new String(p.getInputStream().readAllBytes()) - //log.debug("call.result ${result}") - int exitCode = p.waitFor() - //log.debug("call.exitCode ${exitCode}") - if (exitCode != 0) { - log.debug("`call.fail` rc=${exitCode}[${cmd}]: ${result}\n") - } - return exitCode - } - catch (Exception e) { - log.warn("${e.getMessage()}: `${cmd}` ${this}") - return -1 - } - } - - // usage: quilt3 install [-h] [--registry REGISTRY] [--top-hash TOP_HASH] - // [--dest DEST] [--dest-registry DEST_REGISTRY] [--path PATH] name - Path install() { - int exitCode = call('install', packageName, key_registry(), key_hash(), key_dest()) - if (exitCode != 0) { - log.warn("${exitCode}: ${packageName} failed to install (may not exist)") + log.info("installing $packageName from $bucket...") + S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) + Registry registry = new Registry(registryPath) + Namespace namespace = registry.getNamespace(packageName) + String resolvedHash = (hash == 'latest' || hash == null || hash == 'null') ? namespace.getHash('latest') : hash + log.info("hash: $hash -> $resolvedHash") + Manifest manifest = namespace.getManifest(resolvedHash) + + manifest.install(dest) + log.info("done") + } catch (IOException e) { + log.error("failed to install $packageName") return null } + installed = true recursiveDeleteOnExit() - return packageDest() + + return dest } // https://stackoverflow.com/questions/15022219 @@ -284,10 +234,37 @@ class QuiltPackage { } // https://docs.quiltdata.com/v/version-5.0.x/examples/gitlike#install-a-package int push(String msg = 'update', Map meta = [:]) { - int exitCode = 0 - String args = [key_dir(), key_registry(), key_meta(meta), key_msg(msg), key_force()].join(' ') - exitCode = call('push', packageName, args, key_workflow()) - return exitCode + S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) + Registry registry = new Registry(registryPath) + Namespace namespace = registry.getNamespace(packageName) + + Manifest.Builder builder = Manifest.builder() + + Files.walk(packageDest()).filter(f -> Files.isRegularFile(f)).forEach(f -> { + System.out.println(f) + String logicalKey = packageDest().relativize(f) + LocalPhysicalKey physicalKey = new LocalPhysicalKey(f) + long size = Files.size(f) + builder.addEntry(logicalKey, new Entry(physicalKey, size, null, null)) + }); + + Map fullMeta = [ + "version": Manifest.VERSION, + "user_meta": meta + this.meta, + ] + ObjectMapper mapper = new ObjectMapper() + builder.setMetadata((ObjectNode)mapper.valueToTree(fullMeta)) + + Manifest m = builder.build() + + try { + m.push(namespace, "nf-quilt:${today()}-${msg}", parsed.workflowName) + } catch (WorkflowException e) { + return 1 + } catch (IOException e) { + return 1 + } + return 0 } @Override diff --git a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF index 85555939..ea14aa6c 100644 --- a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF @@ -1,7 +1,7 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.quilt.QuiltPlugin Plugin-Id: nf-quilt -Plugin-Version: 0.6.0 +Plugin-Version: 0.7.0 Plugin-Provider: Quilt Data Plugin-Requires: >=22.10.6 diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy index 0c6b0a92..9ace8dcc 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy @@ -60,7 +60,8 @@ class QuiltSpecification extends Specification { PluginExtensionProvider.reset() // this need to be set *before* the plugin manager class is created fullURL = 'quilt+s3://bkt?key=val&key2=val2' + - '#package=pre/suf@ab&path=p/t&property=prop&workflow=wf&catalog=quiltdata.com' + '#package=pre/suf@abcdef314159265'+ + '&path=p/t&property=prop&workflow=wf&catalog=quiltdata.com' pluginsMode = System.getProperty('pf4j.mode') timestamp = System.currentTimeMillis() writeBucket = System.getenv('WRITE_BUCKET')