Squashed commit of the following:

commit 016c59f Author: Dima Ryazanov <[email protected]> Date: Mon Sep 18 07:16:20 2023 -0700 Bump quiltcore and use workflows commit 843d22b Author: Ernest Prabhakar <[email protected]> Date: Mon Sep 11 15:35:28 2023 -0700 re-enabled failing workflow tests commit 41d6063 Author: Dr. Ernie Prabhakar <[email protected]> Date: Mon Sep 11 15:23:07 2023 -0700 Quiltcore java preview (#122) Ernie's updates for Sep 6-11 - Added unit and integration tests - Rewrote package to NOT pre-install (instead use Manifest to find relativeChildren) --------- Co-authored-by: Dima Ryazanov <[email protected]> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
quiltdata · Sep 18, 2023 · af3dd27 · af3dd27
1 parent 01b268e
commit af3dd27
Show file tree

Hide file tree

Showing 8 changed files with 153 additions and 22 deletions.
diff --git a/.github/workflows/pkg-test.yml b/.github/workflows/pkg-test.yml
@@ -0,0 +1,61 @@
+name: Test
+
+on:
+  # Trigger at every push. Action will also be visible from Pull Requests to master
+  push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions)
+  pull_request:
+    branches: [master]
+
+permissions: read-all
+
+jobs:
+  build:
+    name: Package Test
+    permissions:
+      contents: read
+      id-token: write
+      issues: write
+      pull-requests: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        java_version: [19]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    # Git Checkout
+    - name: Checkout Code
+      uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        fetch-depth: 0
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v3
+      with:
+        role-to-assume: arn:aws:iam::712023778557:role/GitHub-Testing-NF-Quilt
+        aws-region: us-east-1
+
+    - name: Setup Java ${{matrix.java_version}}
+      uses: actions/setup-java@v3
+      with:
+        java-version: ${{matrix.java_version}}
+        distribution: 'temurin'
+        architecture: x64
+        cache: gradle
+
+    - name: Setup Gradle
+      uses: gradle/gradle-build-action@v2
+
+    - name: Run Package Tests
+      run: make pkg-test
+
+    - name: Archive production artifacts
+      if: ${{ success() }} || ${{ failure() }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: nf-quilt-pkg-test
+        path: |
+          /home/runner/work/nf-quilt/nf-quilt/plugins/nf-quilt/build/reports/tests/test/
diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle
@@ -61,7 +61,7 @@ ext{
 
 dependencies {
     // quiltcore
-    implementation 'com.quiltdata.quiltcore:quiltcore:0.0.2'
+    implementation 'com.quiltdata.quiltcore:quiltcore:0.0.3'
 
     // This dependency is exported to consumers, that is to say found on their compile classpath.
     compileOnly "io.nextflow:nextflow:$nextflowVersion"

diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy
@@ -80,11 +80,17 @@ ${meta['workflow']['stats']['processes']}
     static void writeString(String text, QuiltPackage pkg, String filename) {
         String dir = pkg.packageDest()
         Path path  = Paths.get(dir, filename.split('/') as String[])
+        File parent = path.getParent().toFile()
+        if (parent != null && !parent.exists() && !parent.mkdirs()) {
+            throw new IllegalStateException("Couldn't create dir: " + parent);
+        }
+
         try {
             Files.write(path, text.bytes)
         }
         catch (Exception e) {
-            log.error("writeString: cannot write `$text` to `$path` for `${pkg}`")
+            log.error("writeString[${e.getMessage()}]: fail write `$path` for `${pkg}`")
+            e.printStackTrace()
         }
     }
 

diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy
@@ -254,7 +254,7 @@ class QuiltPackage {
         Manifest m = builder.build()
 
         try {
-            m.push(namespace, "nf-quilt:${today()}-${msg}")
+            m.push(namespace, "nf-quilt:${today()}-${msg}", parsed.workflowName)
         } catch (IOException e) {
             return 1
         }

diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy
@@ -43,6 +43,7 @@ import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import nextflow.Global
 import nextflow.Session
+import nextflow.file.FileSystemTransferAware
 import nextflow.quilt.jep.QuiltParser
 import nextflow.quilt.jep.QuiltPackage
 
@@ -54,7 +55,7 @@ import nextflow.quilt.jep.QuiltPackage
 
 @Slf4j
 @CompileStatic
-class QuiltFileSystemProvider extends FileSystemProvider {
+class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTransferAware {
 
     private final Map<String,String> myEnv = new HashMap<>(System.getenv())
     private final Map<String,QuiltFileSystem> fileSystems = [:]
@@ -84,6 +85,36 @@ class QuiltFileSystemProvider extends FileSystemProvider {
         return path.getFileSystem().provider()
     }
 
+    static boolean isLocalProvider(Path path) {
+        FileSystemProvider provider = provider(path)
+        String providerName = provider?.class?.name?.toLowerCase() ?: 'N/A'
+        println("QuiltFileSystemProvider.isLocalProvider[${path}] -> ${providerName}")
+        return providerName.contains("xfile") || providerName.contains("win") ||\
+               providerName.contains("fat") || providerName == 'N/A'
+    }
+
+    boolean canDownload(Path source, Path target) {
+        log.debug("QuiltFileSystemProvider.canDownload[${source}] -> ${target}")
+        return isLocalProvider(target) && source instanceof QuiltPath
+    }
+
+    boolean canUpload(Path source, Path target) {
+        log.debug("QuiltFileSystemProvider.canUpload[${source}] -> ${target}")
+        return isLocalProvider(source) && target instanceof QuiltPath
+    }
+
+    void download(Path source, Path target, CopyOption... options) throws IOException {
+        QuiltPath qSource = asQuiltPath(source)
+        Path local_source = qSource.localPath()
+        Files.copy(local_source, target, options)
+    }
+
+    void upload(Path source, Path target, CopyOption... options) throws IOException {
+        QuiltPath qTarget = asQuiltPath(target)
+        Path local_target = qTarget.localPath()
+        Files.copy(source, local_target, options)
+    }
+
     /**
      * @inheritDoc
      */
@@ -340,7 +371,7 @@ class QuiltFileSystemProvider extends FileSystemProvider {
 
     @Override
     void copy(Path from, Path to, CopyOption... options) throws IOException {
-        //log.debug("Attempting `copy`: ${from} -> ${to}")
+        log.debug("Attempting `copy`: ${from} -> ${to}")
         assert provider(from) == provider(to)
         if (from == to) {
             return // nothing to do -- just return
@@ -366,7 +397,7 @@ class QuiltFileSystemProvider extends FileSystemProvider {
 
     @Override
     boolean isHidden(Path path) throws IOException {
-        return path.getFileName()?.toString()?.startsWith('.')
+        return path.getFileName()?.toString()?.contains('.')
     }
 
     @Override
@@ -400,7 +431,7 @@ class QuiltFileSystemProvider extends FileSystemProvider {
     @Override
     def <A extends BasicFileAttributes> A readAttributes(Path path, Class<A> type, LinkOption... options)
          throws IOException {
-        //log.debug '<A>BasicFileAttributes QuiltFileSystemProvider.readAttributes()'
+        log.debug '<A>BasicFileAttributes QuiltFileSystemProvider.readAttributes()'
         def attr = attributesCache.get(path)
         if (attr) {
             return attr

diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy
@@ -106,6 +106,30 @@ class QuiltPackageTest extends QuiltSpecification {
         Files.exists(installPath)
     }
 
+    void 'should copy temp files into install folder'() {
+        given:
+        String filename = 'test.txt'
+        Path installPath = pkg.packageDest()
+        Path tempFile = File.createTempFile('test', '.txt').toPath()
+        Path installedFile = Paths.get(installPath.toString(), filename)
+        expect:
+        Files.exists(tempFile)
+        Files.exists(installPath)
+        !Files.exists(installedFile)
+        Files.copy(tempFile, installedFile)
+        Files.exists(installedFile)
+    }
+
+    void 'should copy package files to temp Path'() {
+        given:
+        Path installPath = pkg.packageDest()
+        expect:
+        Files.exists(installPath)
+        Files.isDirectory(installPath)
+        Files.readAttributes(installPath, BasicFileAttributes)
+    }
+
+
     void 'should get attributes for package folder'() {
         given:
         def root = qpath.getRoot()
@@ -126,6 +150,7 @@ class QuiltPackageTest extends QuiltSpecification {
         Files.readAttributes(qpath, BasicFileAttributes)
     }
 
+
     void 'should return null on failed install'() {
         given:
         def url2 = TEST_URL.replace('quilt-', 'quilted-')
@@ -140,17 +165,13 @@ class QuiltPackageTest extends QuiltSpecification {
     void 'should deinstall files'() {
         expect:
         Files.exists(qpath.localPath(true))
+        Files.readAttributes(qpath, BasicFileAttributes)
         when:
         qpath.deinstall()
         then:
         !Files.exists(qpath.localPath(false))
-        /* when:
-        Files.readAttributes(qpath, BasicFileAttributes)
-        then:
-        thrown(java.nio.file.NoSuchFileException) */
     }
 
-    @Ignore()
     void 'should iterate over installed files '() {
         given:
         def root = qpath.getRoot()
@@ -212,7 +233,7 @@ class QuiltPackageTest extends QuiltSpecification {
         opkg.push('msg', meta) == 0
     }
 
-    @Ignore('QuiltCore-java does not support workflows yet')
+    // @Ignore('QuiltCore-java does not support workflows yet')
     @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET ==  null })
     void 'should fail if invalid workflow'() {
         given:
@@ -222,7 +243,7 @@ class QuiltPackageTest extends QuiltSpecification {
         bad_wf.push('missing-workflow first time', [:]) == 1
     }
 
-    @Ignore('QuiltCore-java does not support workflows yet')
+    // @Ignore('QuiltCore-java does not support workflows yet')
     @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET ==  null })
     void 'should fail push if unsatisfied workflow'() {
         given:

diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy
@@ -3,6 +3,8 @@ package nextflow.quilt.nio
 
 import nextflow.quilt.QuiltSpecification
 import groovy.transform.CompileDynamic
+import java.nio.file.Path
+import java.nio.file.Paths
 
 /**
  *
@@ -22,4 +24,14 @@ class QuiltFileSystemProviderTest extends QuiltSpecification {
     // newDirectoryStream returns package path for write
     // do we need a new schema for quilt+local?
 
+    void 'should recognize isLocalProvider'() {
+        given:
+        Path local = File.createTempFile('test', '.txt').toPath()
+        Path remote = Paths.get(new URI(fullURL))
+
+        expect:
+        QuiltFileSystemProvider.isLocalProvider(local) == true
+        QuiltFileSystemProvider.isLocalProvider(remote) == false
+    }
+
 }
diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy
@@ -79,7 +79,7 @@ class QuiltNioTest extends QuiltSpecification {
         text.startsWith('id')
     }
 
-    @IgnoreIf({ System.getProperty('os.name').contains('ux') })
+    // @IgnoreIf({ System.getProperty('os.name').contains('ux') })
     @IgnoreIf({ System.getProperty('os.name').contains('indows') })
     void 'should read file attributes'() {
         given:
@@ -132,7 +132,7 @@ class QuiltNioTest extends QuiltSpecification {
         then:
         !attrs.isRegularFile()
         attrs.isDirectory()
-        attrs.size() == 128
+        attrs.size() > 100 // differs by platform
         !attrs.isSymbolicLink()
         !attrs.isOther()
         attrs.fileKey() == root
@@ -148,7 +148,7 @@ class QuiltNioTest extends QuiltSpecification {
         then:
         !attrs.isRegularFile()
         attrs.isDirectory()
-        attrs.size() == 224
+        attrs.size() > 100 // differs by platform
         !attrs.isSymbolicLink()
         !attrs.isOther()
         attrs.fileKey() == '/'
@@ -197,7 +197,7 @@ class QuiltNioTest extends QuiltSpecification {
         if (source) { Files.delete(source) }
     }
 
-    @Ignore
+    @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET ==  null })
     void 'copy a remote file to a bucket'() {
         given:
         Path path = Paths.get(new URI(WRITE_URL))
@@ -212,11 +212,11 @@ class QuiltNioTest extends QuiltSpecification {
         readObject(path).trim() == TEXT
     }
 
-    @Ignore
+    @Ignore('QuiltFileSystem.copy not implemented')
     void 'move a remote file to a bucket'() {
         given:
         Path path = Paths.get(new URI(WRITE_URL))
-        final source_url = WRITE_URL.replace('test_folder', 'source')
+        final source_url = WRITE_URL.replace('folder', 'source')
         final source = Paths.get(new URI(source_url))
         Files.write(source, TEXT.bytes)
         and:
@@ -471,7 +471,7 @@ class QuiltNioTest extends QuiltSpecification {
         thrown(FileSystemException)
     }
 
-    @Ignore
+    @Ignore('Can not write to null_path')
     void 'should stream directory content'() {
         given:
         makeObject(null_path('foo/file1.txt'), 'A')
@@ -515,7 +515,7 @@ class QuiltNioTest extends QuiltSpecification {
         list  == [ 'file4.txt' ]
     }
 
-    @Ignore
+    @Ignore('Can not write to null_path')
     void 'should check walkTree'() {
         given:
         makeObject(null_path('foo/file1.txt'), 'A')