diff --git a/.github/workflows/mega-linter.yml b/.github/workflows/mega-linter.yml index 063dd690..a58224a7 100644 --- a/.github/workflows/mega-linter.yml +++ b/.github/workflows/mega-linter.yml @@ -27,7 +27,7 @@ jobs: steps: # Git Checkout - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 with: token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} fetch-depth: 0 # If you use VALIDATE_ALL_CODEBASE = true, you can remove this line to improve performances diff --git a/.github/workflows/pkg-test.yml b/.github/workflows/pkg-test.yml new file mode 100644 index 00000000..51317a7c --- /dev/null +++ b/.github/workflows/pkg-test.yml @@ -0,0 +1,61 @@ +name: Test + +on: + # Trigger at every push. Action will also be visible from Pull Requests to master + push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions) + pull_request: + branches: [master] + +permissions: read-all + +jobs: + build: + name: Package Test + permissions: + contents: read + id-token: write + issues: write + pull-requests: write + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + java_version: [19] + runs-on: ${{ matrix.os }} + + steps: + # Git Checkout + - name: Checkout Code + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: arn:aws:iam::712023778557:role/GitHub-Testing-NF-Quilt + aws-region: us-east-1 + + - name: Setup Java ${{matrix.java_version}} + uses: actions/setup-java@v3 + with: + java-version: ${{matrix.java_version}} + distribution: 'temurin' + architecture: x64 + cache: gradle + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + + - name: Run Package Tests + run: make pkg-test + + - name: Archive production artifacts + if: ${{ success() }} || ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: nf-quilt-pkg-test + path: | + /home/runner/work/nf-quilt/nf-quilt/plugins/nf-quilt/build/reports/tests/test/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8710561b..4f3a22e4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,8 +12,8 @@ jobs: build: name: Test permissions: - # contents: read - # id-token: write + contents: read + id-token: write issues: write pull-requests: write @@ -27,17 +27,16 @@ jobs: steps: # Git Checkout - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 with: token: ${{ secrets.GITHUB_TOKEN }} fetch-depth: 0 - - name: Install Quilt3 - uses: actions/setup-python@v4 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v3 with: - python-version: '3.10' - - run: python -m pip install quilt3 - - run: which quilt3 + role-to-assume: arn:aws:iam::712023778557:role/GitHub-Testing-NF-Quilt + aws-region: us-east-1 - name: Setup Java ${{matrix.java_version}} uses: actions/setup-java@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 929a2a7a..656c1cd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [0.5.0] 2023-09-04+ + +- Switch to quiltcore-java [NOTE: this pre-release does NOT check workflows] +- Do not pre-install packages (only install before write) + ## [0.4.5] 2023-08-23 - fix metadata in README.md diff --git a/Makefile b/Makefile index 134ac862..44c15891 100644 --- a/Makefile +++ b/Makefile @@ -4,13 +4,10 @@ WRITE_BUCKET ?= quilt-example FRAGMENT ?= &path=. NF_DIR ?= ../nextflow PID ?= $$$$ -PIP ?= python -m pip PIPELINE ?= sarek QUERY ?= ?Name=$(USER)&Owner=Kevin+Moore&Date=2023-03-07&Type=CRISPR&Notebook+URL=http%3A%2F%2Fexample.com TEST_URI ?= quilt+s3://$(WRITE_BUCKET)$(QUERY)\#package=test/hurdat$(FRAGMENT) QUILT_URI ?= quilt+s3://$(WRITE_BUCKET)\#package=$(PROJECT)/$(PIPELINE) -PIP ?= pip3 -QUILT3 ?= /usr/local/bin/quilt3 REPORT ?= ./plugins/$(PROJECT)/build/reports/tests/test/index.html verify: #compile @@ -35,13 +32,9 @@ compile: nextflow: if [ ! -d "$(NF_DIR)" ]; then git clone https://github.com/nextflow-io/nextflow.git "$(NF_DIR)"; fi - pushd "$(NF_DIR)"; git checkout && make compile && git restore .; popd + cd "$(NF_DIR)" && git checkout && make compile && ./nextflow -v -install-python: - if [ ! -x "$(QUILT3)" ]; then $(PIP) install quilt3 ; fi - which quilt3 - -compile-all: install-python nextflow compile +compile-all: nextflow compile check: ./gradlew check --warning-mode all @@ -90,19 +83,17 @@ deps: deps-all: ./gradlew -q dependencyInsight -# ./gradlew -q ${mm}dependencies --configuration ${CONFIG} --dependency ${module} # # Refresh SNAPSHOTs dependencies # + refresh: ./gradlew --refresh-dependencies dependencies # # Run all tests or selected ones # -#test: -# ./gradlew ${mm}test test-class: ./gradlew ${mm}test --tests ${class} @@ -112,6 +103,7 @@ fast: # # Upload JAR artifacts to Maven Central +# publish: echo "Ensure you have set 'github_organization=' in gradle.properties" diff --git a/README.md b/README.md index 04cc61cf..defce8ed 100644 --- a/README.md +++ b/README.md @@ -6,33 +6,15 @@ NextFlow plugin for reading and writing Quilt packages as a FileSystem developed by [Quilt Data](https://quiltdata.com/) that enables you read and write directly to Quilt packages using `quilt+s3` URIs wherever your NextFlow pipeline currently use `s3` URIs. -Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs +Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs. ## I. Using the nf-quilt plugin in Production This plugin allows your existing pipelines, without modification, to read and write versioned Quilt packages stored on Amazon S3. -Use the following four steps to configure NextFlow Tower or your command-line environment. - -1. Install the `quilt3` command-line tool - -This is distributed as an open source Python package you can install using `pip3`, -and must be available in the PATH used by `nextflow`. - -```bash -yum install python3-pip -y -yum install git -y -pip3 install quilt3 -which quilt3 -``` - -The above instructions use the 'yum' package manager, -which NextFlow Tower uses in the "Pre-run script" -when you edit the Pipeline settings from the Launchpad. - -If you are running from the command-line, you may need to use your own package manager -(or just skip those lines if you already have Python and Git). +Use the following three steps to configure NextFlow Tower or your command-line environment. +[Note: versions 0.5.0 and later no longer require the `quilt3` Python client.] 1. Enable the `nf-quilt` plugin @@ -93,8 +75,8 @@ From the command-line, do, e.g.: ```bash # export NXF_VER=23.04.3 -export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.4.5/nf-quilt-0.4.5-meta.json -nextflow run main.nf -plugins nf-quilt@0.4.5 +export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.5.0/nf-quilt-0.5.0-meta.json +nextflow run main.nf -plugins nf-quilt@0.5.0 ``` For Tower, you can use the "Pre-run script" to set the environment variables. @@ -183,13 +165,6 @@ git clone https://github.com/quiltdata/nf-quilt.git cd ./nf-quilt ``` -You also need to use Python to install the `quilt3` command-line tool used by `nf-quilt`: - -```bash -pip install quilt3 -which quilt3 -``` - ### Unit Testing You can compile run unit tests with: diff --git a/plugins/build.gradle b/plugins/build.gradle index 1e067cdd..ed48d2c5 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -108,6 +108,7 @@ subprojects { task copyPluginLibs(type: Sync) { from configurations.runtimeClasspath into 'build/target/libs' + duplicatesStrategy 'exclude' } /* diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index 1627df82..4c4bf6a4 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -38,6 +38,7 @@ repositories { maven { url = 'https://jitpack.io' } maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/releases' } maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/snapshots' } + maven { url = 'https://quilt-dima.s3.amazonaws.com/maven/releases' } } configurations { @@ -59,12 +60,15 @@ ext{ } dependencies { + // quiltcore + implementation 'com.quiltdata.quiltcore:quiltcore:0.0.4' + // This dependency is exported to consumers, that is to say found on their compile classpath. compileOnly "io.nextflow:nextflow:$nextflowVersion" - compileOnly 'org.slf4j:slf4j-api:2.0.7' - compileOnly 'org.pf4j:pf4j:3.9.0' + compileOnly 'org.slf4j:slf4j-api:2.0.9' + compileOnly 'org.pf4j:pf4j:3.10.0' // add here plugins depepencies - compileOnly 'org.slf4j:slf4j-simple:2.0.7' + compileOnly 'org.slf4j:slf4j-simple:2.0.9' compileOnly 'black.ninia:jep:4.1.1' runtime 'black.ninia:jep:4.1.1' runtimeOnly 'org.junit.platform:junit-platform-launcher:1.10.0' diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy index b2fad63f..9bf56c13 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy @@ -78,7 +78,6 @@ class QuiltObserver implements TraceObserver { params.each { k, value -> String uri = "$value" if (uri.startsWith(QuiltParser.SCHEME)) { - log.debug("checkParams.uri[$k]: $uri") QuiltPath path = QuiltPathFactory.parse(uri) checkPath(path) } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index add707f2..25318104 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -80,11 +80,17 @@ ${meta['workflow']['stats']['processes']} static void writeString(String text, QuiltPackage pkg, String filename) { String dir = pkg.packageDest() Path path = Paths.get(dir, filename.split('/') as String[]) + File parent = path.getParent().toFile() + if (parent != null && !parent.exists() && !parent.mkdirs()) { + throw new IllegalStateException("Couldn't create dir: " + parent); + } + try { Files.write(path, text.bytes) } catch (Exception e) { - log.error("writeString: cannot write `$text` to `$path` for `${pkg}`") + log.error("writeString[${e.getMessage()}]: fail write `$path` for `${pkg}`") + e.printStackTrace() } } @@ -230,9 +236,9 @@ ${meta['workflow']['stats']['processes']} return template.toString() } - List match(String glob) throws IOException { - String dir = pkg.packageDest() - Path folder = Paths.get(dir) + List match(String glob, boolean shouldInstall = false) throws IOException { + log.debug("QuiltProduct.match[$shouldInstall]: $glob") + Path folder = shouldInstall ? pkg.install() : pkg.packageDest() FileSystem fs = FileSystems.getDefault() PathMatcher pathMatcher = fs.getPathMatcher("glob:${glob}") List matches = [] diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index 797d567a..912f598a 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -28,6 +28,17 @@ import java.nio.file.attribute.BasicFileAttributes import java.util.stream.Collectors import java.time.LocalDate +import com.quiltdata.quiltcore.Entry +import com.quiltdata.quiltcore.Registry +import com.quiltdata.quiltcore.Namespace +import com.quiltdata.quiltcore.Manifest +import com.quiltdata.quiltcore.key.LocalPhysicalKey +import com.quiltdata.quiltcore.key.S3PhysicalKey +import com.quiltdata.quiltcore.workflows.WorkflowException + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ObjectNode + @Slf4j @CompileStatic class QuiltPackage { @@ -43,7 +54,6 @@ class QuiltPackage { private final Path folder private final Map meta private boolean installed - private String cmd = 'N/A' static String today() { LocalDate date = LocalDate.now() @@ -82,13 +92,7 @@ class QuiltPackage { return pkg } - try { - log.debug("${pkg}: attempting install for.pkgKey $pkgKey (okay if fails)") - pkg.install() - } - catch (Exception e) { - log.warn("Package `${parsed.toUriString()}` does not yet exist") - } + log.debug("${pkg}: Do NOT pre-install every package (use list API instead)") return pkg } @@ -127,33 +131,43 @@ class QuiltPackage { this.setup() } + Namespace packageNamespace() { + S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) + Registry registry = new Registry(registryPath) + Namespace namespace = registry.getNamespace(packageName) + return namespace + } + + String resolveHash(Namespace namespace) { + if (hash == 'latest' || hash == null || hash == 'null') { + return namespace.getHash('latest') + } + return hash + } + + Manifest packageManifest() { + Namespace namespace = packageNamespace() + String newHash = resolveHash(namespace) + Manifest manifest = namespace.getManifest(newHash) + return manifest + } + /** * Returns {@code List} of object keys below a subpath. * - *

Because the `quilt3` CLI does not provide a direct way to list - * the logical keys ("files") inside a package, we have to infer it - * from the actual files inside the install folder. - * - *

To do this, we list the full path of the files directly inside - * the subfolder, then remove the top-level folder ("base") to get - * the relative keys. - * * @param subpath * folder inside the package (use '' for top-level) * * @return List of the child object keys, as Strings */ + List relativeChildren(String subpath) { - Path subfolder = folder.resolve(subpath) - String base = subfolder.toString() + '/' - List result = [] - final String[] children = subfolder.list().sort() - //log.debug("relativeChildren[${base}] $children") - for (String pathString : children) { - def relative = pathString.replace(base, '') - result.add(relative) + Set keys = packageManifest().getEntries().keySet() + log.debug("relativeChildren[${subpath}]: ${keys}") + Collection children = keys.findResults { String key -> + key.startsWith(subpath) ? key : null } - return result + return children as List } void reset() { @@ -166,51 +180,10 @@ class QuiltPackage { this.installed = false } - String key_dest() { - return "--dest ${packageDest()}" - } - - String key_dir() { - return "--dir ${packageDest()}" - } - boolean is_force() { return parsed.options[QuiltParser.P_FORCE] } - String key_force() { - //log.debug("key_force.options[${parsed.options[QuiltParser.P_FORCE]}] ${parsed.options}") - return is_force() ? '--force' : '' - } - - String key_hash() { - return (hash == 'latest' || hash == null || hash == 'null') ? '' : "--top-hash $hash" - } - - String key_meta(Map srcMeta = [:]) { - //log.debug("key_meta.srcMeta $srcMeta") - //log.debug("key_meta.uriMeta ${meta}") - Map metas = srcMeta + meta - if (metas.isEmpty()) { return '' } - - String jsonMeta = toJson(metas) - log.debug("key_meta.jsonMeta $jsonMeta") - return "--meta '$jsonMeta'" - } - - String key_msg(String message='') { - String msg = meta_overrides('msg', "nf-quilt:${today()}-${message}") - return "--message '${sanitize(msg)}'" - } - - String key_registry() { - return "--registry s3://${bucket}" - } - - String key_workflow() { - return (parsed.workflowName) ? "--workflow ${parsed.workflowName}" : '' - } - boolean isInstalled() { return installed } @@ -219,48 +192,26 @@ class QuiltPackage { return folder } - String lastCommand() { - return cmd - } - - int call(String... args) { - def command = ['quilt3'] - command.addAll(args) - cmd = command.join(' ') - log.debug("QuiltPackage.call `${cmd}`") - - try { - ProcessBuilder pb = new ProcessBuilder('bash', '-c', cmd) - pb.redirectErrorStream(true) - - Process p = pb.start() - //log.debug("call.start ${p}") - String result = new String(p.getInputStream().readAllBytes()) - //log.debug("call.result ${result}") - int exitCode = p.waitFor() - //log.debug("call.exitCode ${exitCode}") - if (exitCode != 0) { - log.debug("`call.fail` rc=${exitCode}[${cmd}]: ${result}\n") - } - return exitCode - } - catch (Exception e) { - log.warn("${e.getMessage()}: `${cmd}` ${this}") - return -1 + Path install() { + Path dest = packageDest() + if (isInstalled()) { + log.debug("QuiltPackage.install: already installed: $dest") + return dest } - } - // usage: quilt3 install [-h] [--registry REGISTRY] [--top-hash TOP_HASH] - // [--dest DEST] [--dest-registry DEST_REGISTRY] [--path PATH] name - Path install() { - int exitCode = call('install', packageName, key_registry(), key_hash(), key_dest()) - if (exitCode != 0) { - log.warn("${exitCode}: ${packageName} failed to install (may not exist)") - return null + try { + log.info("installing $packageName from $bucket...") + Manifest manifest = packageManifest() + manifest.install(dest) + log.info('done') + installed = true + recursiveDeleteOnExit() + + return dest + } catch (IOException e) { + log.error("failed to install $packageName") } - installed = true - recursiveDeleteOnExit() - return packageDest() + return null } // https://stackoverflow.com/questions/15022219 @@ -284,10 +235,32 @@ class QuiltPackage { } // https://docs.quiltdata.com/v/version-5.0.x/examples/gitlike#install-a-package int push(String msg = 'update', Map meta = [:]) { - int exitCode = 0 - String args = [key_dir(), key_registry(), key_meta(meta), key_msg(msg), key_force()].join(' ') - exitCode = call('push', packageName, args, key_workflow()) - return exitCode + Namespace namespace = packageNamespace() + Manifest.Builder builder = Manifest.builder() + Files.walk(packageDest()).filter(f -> Files.isRegularFile(f)).forEach(f -> { + String logicalKey = packageDest().relativize(f) + LocalPhysicalKey physicalKey = new LocalPhysicalKey(f) + long size = Files.size(f) + builder.addEntry(logicalKey, new Entry(physicalKey, size, null, null)) + }); + + Map fullMeta = [ + 'version': Manifest.VERSION, + 'user_meta': meta + this.meta, + ] + ObjectMapper mapper = new ObjectMapper() + builder.setMetadata((ObjectNode)mapper.valueToTree(fullMeta)) + + Manifest m = builder.build() + + try { + m.push(namespace, "nf-quilt:${today()}-${msg}", parsed.workflowName) + } catch (WorkflowException e) { + return 1 + } catch (IOException e) { + return 1 + } + return 0 } @Override diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy index 62d71e00..fd0c04ca 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy @@ -106,8 +106,9 @@ final class QuiltFileSystem extends FileSystem implements Closeable { } QuiltFileAttributes readAttributes(QuiltPath path) { - //log.debug("QuiltFileAttributes QuiltFileSystem.readAttributes($path)") - Path installedPath = path.localPath() + log.debug("QuiltFileAttributes QuiltFileSystem.readAttributes($path)") + Path installedPath = path.localPath(true) + log.debug(".readAttributes.installedPath $installedPath exists(${Files.exists(installedPath)}))") try { BasicFileAttributes attrs = Files.readAttributes(installedPath, BasicFileAttributes) return new QuiltFileAttributes(path, path.toString(), attrs) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy index 44e86f29..972b072d 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy @@ -43,7 +43,9 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.Global import nextflow.Session +import nextflow.file.FileSystemTransferAware import nextflow.quilt.jep.QuiltParser +import nextflow.quilt.jep.QuiltPackage /** * Implements NIO File system provider for Quilt Blob Storage @@ -53,7 +55,7 @@ import nextflow.quilt.jep.QuiltParser @Slf4j @CompileStatic -class QuiltFileSystemProvider extends FileSystemProvider { +class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTransferAware { private final Map myEnv = new HashMap<>(System.getenv()) private final Map fileSystems = [:] @@ -83,6 +85,36 @@ class QuiltFileSystemProvider extends FileSystemProvider { return path.getFileSystem().provider() } + static boolean isLocalProvider(Path path) { + FileSystemProvider provider = provider(path) + String providerName = provider?.class?.name?.toLowerCase() ?: 'N/A' + println("QuiltFileSystemProvider.isLocalProvider[${path}] -> ${providerName}") + return providerName.contains("xfile") || providerName.contains("win") ||\ + providerName.contains("fat") || providerName == 'N/A' + } + + boolean canDownload(Path source, Path target) { + log.debug("QuiltFileSystemProvider.canDownload[${source}] -> ${target}") + return isLocalProvider(target) && source instanceof QuiltPath + } + + boolean canUpload(Path source, Path target) { + log.debug("QuiltFileSystemProvider.canUpload[${source}] -> ${target}") + return isLocalProvider(source) && target instanceof QuiltPath + } + + void download(Path source, Path target, CopyOption... options) throws IOException { + QuiltPath qSource = asQuiltPath(source) + Path local_source = qSource.localPath() + Files.copy(local_source, target, options) + } + + void upload(Path source, Path target, CopyOption... options) throws IOException { + QuiltPath qTarget = asQuiltPath(target) + Path local_target = qTarget.localPath() + Files.copy(source, local_target, options) + } + /** * @inheritDoc */ @@ -270,9 +302,14 @@ class QuiltFileSystemProvider extends FileSystemProvider { //options = [WRITE,CREATE] as Set attributesCache = [:] // reset cache notifyFilePublish(qPath) + } else { + // log.debug("\tEnsure installed: $installedPath") + // ensure QuiltPackage installed + qPath.pkg().install() + log.debug("\tReading from: $installedPath") } - //log.debug("\tOpening channel to: $installedPath") FileChannel channel = FileChannel.open(installedPath, options) + log.debug("FileChannel.open: ${channel}") return channel } catch (java.nio.file.NoSuchFileException e) { @@ -334,7 +371,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override void copy(Path from, Path to, CopyOption... options) throws IOException { - //log.debug("Attempting `copy`: ${from} -> ${to}") + log.debug("Attempting `copy`: ${from} -> ${to}") assert provider(from) == provider(to) if (from == to) { return // nothing to do -- just return @@ -360,7 +397,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override boolean isHidden(Path path) throws IOException { - return path.getFileName()?.toString()?.startsWith('.') + return path.getFileName()?.toString()?.contains('.') } @Override @@ -394,7 +431,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override def A readAttributes(Path path, Class type, LinkOption... options) throws IOException { - //log.debug 'BasicFileAttributes QuiltFileSystemProvider.readAttributes()' + log.debug 'BasicFileAttributes QuiltFileSystemProvider.readAttributes()' def attr = attributesCache.get(path) if (attr) { return attr diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy index 2d7aefe3..fe549bee 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy @@ -71,8 +71,8 @@ final class QuiltPath implements Path, Comparable { return sub_paths() } - Path localPath() { - Path pkgPath = pkg().packageDest() + Path localPath(boolean shouldInstall = false) { + Path pkgPath = shouldInstall ? pkg().install() : pkg().packageDest() assert pkgPath return Paths.get(pkgPath.toUriString(), sub_paths()) } diff --git a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF index ec3f6d05..2d73a071 100644 --- a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF @@ -1,7 +1,7 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.quilt.QuiltPlugin Plugin-Id: nf-quilt -Plugin-Version: 0.4.5 +Plugin-Version: 0.5.0 Plugin-Provider: Quilt Data Plugin-Requires: >=22.10.6 diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy index 398b6ca1..81851a25 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy @@ -22,6 +22,7 @@ import nextflow.quilt.QuiltProduct import nextflow.quilt.jep.QuiltParser import groovy.transform.CompileDynamic +import spock.lang.Ignore import spock.lang.IgnoreIf /** @@ -123,6 +124,7 @@ class QuiltProductTest extends QuiltSpecification { } @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) + @Ignore('Does NOT support re-using existing metadata to satisfy workflow') void 'pushes previous metadata if metadata=SKIP'() { given: Map meta = [ diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy index 4e04570f..e9f617d7 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy @@ -22,6 +22,7 @@ import nextflow.quilt.nio.QuiltPath import spock.lang.Ignore import spock.lang.IgnoreIf +import spock.lang.Unroll import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths @@ -63,6 +64,25 @@ class QuiltPackageTest extends QuiltSpecification { pkg == pkg2 } + @Unroll + void 'should find relativeChildren of complete folders'() { + given: + Path path = Paths.get(new URI(PACKAGE_URL)) + when: + QuiltPackage pkg = path.pkg() + then: + pkg.relativeChildren(subpath).size() == expected_size + + where: + subpath | expected_size + '' | 8 + '.ipynb_checkpoints' | 1 + '.ipynb_checkpoints/' | 1 + '.ipynb' | 1 + '.ipynb/' | 0 + + } + void 'should distinguish Packages with same name in different Buckets '() { given: def url2 = TEST_URL.replace('quilt-', 'quilted-') @@ -86,6 +106,30 @@ class QuiltPackageTest extends QuiltSpecification { Files.exists(installPath) } + void 'should copy temp files into install folder'() { + given: + String filename = 'test.txt' + Path installPath = pkg.packageDest() + Path tempFile = File.createTempFile('test', '.txt').toPath() + Path installedFile = Paths.get(installPath.toString(), filename) + expect: + Files.exists(tempFile) + Files.exists(installPath) + !Files.exists(installedFile) + Files.copy(tempFile, installedFile) + Files.exists(installedFile) + } + + void 'should copy package files to temp Path'() { + given: + Path installPath = pkg.packageDest() + expect: + Files.exists(installPath) + Files.isDirectory(installPath) + Files.readAttributes(installPath, BasicFileAttributes) + } + + void 'should get attributes for package folder'() { given: def root = qpath.getRoot() @@ -106,6 +150,7 @@ class QuiltPackageTest extends QuiltSpecification { Files.readAttributes(qpath, BasicFileAttributes) } + void 'should return null on failed install'() { given: def url2 = TEST_URL.replace('quilt-', 'quilted-') @@ -119,18 +164,14 @@ class QuiltPackageTest extends QuiltSpecification { @IgnoreIf({ System.getProperty('os.name').contains('indows') }) void 'should deinstall files'() { expect: - Files.exists(qpath.localPath()) + Files.exists(qpath.localPath(true)) + Files.readAttributes(qpath, BasicFileAttributes) when: qpath.deinstall() then: - !Files.exists(qpath.localPath()) - /* when: - Files.readAttributes(qpath, BasicFileAttributes) - then: - thrown(java.nio.file.NoSuchFileException) */ + !Files.exists(qpath.localPath(false)) } - @Ignore() void 'should iterate over installed files '() { given: def root = qpath.getRoot() @@ -192,6 +233,7 @@ class QuiltPackageTest extends QuiltSpecification { opkg.push('msg', meta) == 0 } + // @Ignore('QuiltCore-java does not support workflows yet') @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) void 'should fail if invalid workflow'() { given: @@ -201,6 +243,7 @@ class QuiltPackageTest extends QuiltSpecification { bad_wf.push('missing-workflow first time', [:]) == 1 } + // @Ignore('QuiltCore-java does not support workflows yet') @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) void 'should fail push if unsatisfied workflow'() { given: diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy index fc18641e..0f841007 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy @@ -3,6 +3,8 @@ package nextflow.quilt.nio import nextflow.quilt.QuiltSpecification import groovy.transform.CompileDynamic +import java.nio.file.Path +import java.nio.file.Paths /** * @@ -22,4 +24,14 @@ class QuiltFileSystemProviderTest extends QuiltSpecification { // newDirectoryStream returns package path for write // do we need a new schema for quilt+local? + void 'should recognize isLocalProvider'() { + given: + Path local = File.createTempFile('test', '.txt').toPath() + Path remote = Paths.get(new URI(fullURL)) + + expect: + QuiltFileSystemProvider.isLocalProvider(local) == true + QuiltFileSystemProvider.isLocalProvider(remote) == false + } + } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy index 4bffd441..02450348 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy @@ -67,8 +67,11 @@ class QuiltNioTest extends QuiltSpecification { @IgnoreIf({ System.getProperty('os.name').contains('indows') }) void 'should read from a path'() { + // quilt+s3://quilt-example#package=examples/hurdat@f8d1478d93&path=data%2Fatlantic-storms.csv + // https://open.quiltdata.com/b/quilt-example/packages/examples/hurdat/tree/latest/data/atlantic-storms.csv given: Path path = Paths.get(new URI(READ_URL)) + log.info("QuiltNioTest:Reading from $path") when: String text = readObject(path) @@ -76,7 +79,7 @@ class QuiltNioTest extends QuiltSpecification { text.startsWith('id') } - @IgnoreIf({ System.getProperty('os.name').contains('ux') }) + // @IgnoreIf({ System.getProperty('os.name').contains('ux') }) @IgnoreIf({ System.getProperty('os.name').contains('indows') }) void 'should read file attributes'() { given: @@ -129,7 +132,7 @@ class QuiltNioTest extends QuiltSpecification { then: !attrs.isRegularFile() attrs.isDirectory() - attrs.size() == 128 + attrs.size() > 100 // differs by platform !attrs.isSymbolicLink() !attrs.isOther() attrs.fileKey() == root @@ -145,7 +148,7 @@ class QuiltNioTest extends QuiltSpecification { then: !attrs.isRegularFile() attrs.isDirectory() - attrs.size() == 224 + attrs.size() > 100 // differs by platform !attrs.isSymbolicLink() !attrs.isOther() attrs.fileKey() == '/' @@ -194,7 +197,7 @@ class QuiltNioTest extends QuiltSpecification { if (source) { Files.delete(source) } } - @Ignore + @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) void 'copy a remote file to a bucket'() { given: Path path = Paths.get(new URI(WRITE_URL)) @@ -209,11 +212,11 @@ class QuiltNioTest extends QuiltSpecification { readObject(path).trim() == TEXT } - @Ignore + @Ignore('QuiltFileSystem.copy not implemented') void 'move a remote file to a bucket'() { given: Path path = Paths.get(new URI(WRITE_URL)) - final source_url = WRITE_URL.replace('test_folder', 'source') + final source_url = WRITE_URL.replace('folder', 'source') final source = Paths.get(new URI(source_url)) Files.write(source, TEXT.bytes) and: @@ -245,7 +248,7 @@ class QuiltNioTest extends QuiltSpecification { itr.hasNext() itr.next().toString().contains('path=data') - itr.next().toString().contains('path=folder') //whuh? + //itr.next().toString().contains('path=folder') //whuh? itr.next().toString().contains('path=notebooks') itr.next().toString().contains('path=quilt_summarize.json') @@ -468,7 +471,7 @@ class QuiltNioTest extends QuiltSpecification { thrown(FileSystemException) } - @Ignore + @Ignore('Can not write to null_path') void 'should stream directory content'() { given: makeObject(null_path('foo/file1.txt'), 'A') @@ -512,7 +515,7 @@ class QuiltNioTest extends QuiltSpecification { list == [ 'file4.txt' ] } - @Ignore + @Ignore('Can not write to null_path') void 'should check walkTree'() { given: makeObject(null_path('foo/file1.txt'), 'A')