From 97c7702bc2852fe85dc34ec9e66440bf68be5b09 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Mon, 20 Nov 2023 09:52:42 +0100 Subject: [PATCH 1/9] post merge changes --- CHANGELOG.md | 4 ++++ build.sbt | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a46f6fa62..f7121095b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# Viash 0.x.x (yyyy-MM-dd): TODO Add title + +TODO add summary + # Viash 0.8.1 (2023-11-20): Minor bug fix to Nextflow workflows This release fixes a bug in the Nextflow platform where calling a workflow with the `.run()` function without specifying the `fromState` argument would result in an error when the input channel contained tuples with more than two elements. diff --git a/build.sbt b/build.sbt index ec1514f19..a09187698 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ name := "viash" -version := "0.8.1" +version := "0.8.2dev" scalaVersion := "2.13.10" From 0b4caded893198406e9565d07c4107bbf7d35348 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Fri, 1 Dec 2023 16:14:53 +0100 Subject: [PATCH 2/9] Test different versions of nextflow (#593) * test different versions of nextflow * update changelog * update setup-java * use Java 21 LTS instead of Java 20 Co-authored-by: Hendrik Cannoodt * Revert "use Java 21 LTS instead of Java 20" This reverts commit acff5fc34dc36e39ce08e128fb7ff5f80cff4c49. --------- Co-authored-by: Hendrik Cannoodt --- .github/workflows/ns_test.yml | 2 +- .github/workflows/sbt_test.yml | 17 +++++++---------- CHANGELOG.md | 8 ++++++++ .../platforms/nextflow/NextflowTestHelper.scala | 4 ++-- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ns_test.yml b/.github/workflows/ns_test.yml index d3d1b1f52..336f786cf 100644 --- a/.github/workflows/ns_test.yml +++ b/.github/workflows/ns_test.yml @@ -10,7 +10,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up sbt - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: temurin java-version: '11' diff --git a/.github/workflows/sbt_test.yml b/.github/workflows/sbt_test.yml index 8b58b1a03..33ea5f28a 100644 --- a/.github/workflows/sbt_test.yml +++ b/.github/workflows/sbt_test.yml @@ -14,9 +14,9 @@ jobs: - { name: 'ubuntu_latest', os: ubuntu-latest } - { name: 'macos_latest', os: macos-latest } java: - - { ver: '11', run_nextflow: true, run_coverage: false } - - { ver: '17', run_nextflow: true, run_coverage: true } - - { ver: '20', run_nextflow: false, run_coverage: false } + - { ver: '11', run_nextflow: true, run_coverage: false, nxf_ver: "22.04.5" } + - { ver: '17', run_nextflow: true, run_coverage: true, nxf_ver: latest } + - { ver: '20', run_nextflow: true, run_coverage: false, nxf_ver: latest-edge } steps: - uses: actions/checkout@v4 @@ -26,12 +26,9 @@ jobs: - name: Set up Nextflow if: ${{ runner.os == 'Linux' && matrix.java.run_nextflow }} - run: | - mkdir -p "$HOME/.local/bin" - echo "$HOME/.local/bin" >> $GITHUB_PATH - curl -s https://get.nextflow.io 2> /dev/null | bash > /dev/null - mv nextflow "$HOME/.local/bin" - nextflow -version + uses: nf-core/setup-nextflow@v1 + with: + version: ${{ matrix.java.nxf_ver }} - name: Set up R uses: r-lib/actions/setup-r@v2 @@ -46,7 +43,7 @@ jobs: testthat - name: Set up sbt - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: temurin java-version: ${{ matrix.java.ver }} diff --git a/CHANGELOG.md b/CHANGELOG.md index f7121095b..2cc962e01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ TODO add summary +## MINOR CHANGES + +* `NextflowTestHelper`: Do not hardcode a version of Nextflow in the testbench, + but use the version of Nextflow that is installed on the system (PR #593). + +* GitHub Actions: Test different versions of Nextflow (22.04.5, latest, and latest-edge) (PR #593). + Testing the latest Edge version of Nextflow will allow us to catch notice changes in Nextflow earlier. + # Viash 0.8.1 (2023-11-20): Minor bug fix to Nextflow workflows This release fixes a bug in the Nextflow platform where calling a workflow with the `.run()` function without specifying the `fromState` argument would result in an error when the input channel contained tuples with more than two elements. diff --git a/src/test/scala/io/viash/platforms/nextflow/NextflowTestHelper.scala b/src/test/scala/io/viash/platforms/nextflow/NextflowTestHelper.scala index 8a2e43b97..762713be2 100644 --- a/src/test/scala/io/viash/platforms/nextflow/NextflowTestHelper.scala +++ b/src/test/scala/io/viash/platforms/nextflow/NextflowTestHelper.scala @@ -150,9 +150,9 @@ object NextflowTestHelper { args // fix nextflow version to certain release - val extraEnv_ = extraEnv :+ ("NXF_VER" -> "22.04.5") + // val extraEnv_ = extraEnv :+ ("NXF_VER" -> "22.04.5") - val exitCode = Process(command, cwd, extraEnv_ : _*).!( + val exitCode = Process(command, cwd, extraEnv : _*).!( ProcessLogger(str => stdOut ++= s"$str\n", str => stdErr ++= s"$str\n") ) From 1e3db7ddaf79e0918f338138c2900f38abc1080c Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Mon, 11 Dec 2023 12:08:59 +0100 Subject: [PATCH 3/9] add contributing guidelines (#600) * add contributing guidelines * update changelog * add more templates * Update CONTRIBUTING.md Co-authored-by: Hendrik Cannoodt * add staging command Co-authored-by: Hendrik Cannoodt * uniformize contributing guidelines * update contributing info --------- Co-authored-by: Hendrik Cannoodt --- .github/ISSUE_TEMPLATE/bug_report.yaml | 106 +++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.yaml | 70 +++++++++++++ .github/pull_request_template.md | 51 ++++++--- CHANGELOG.md | 9 ++ CONTRIBUTING.md | 108 ++++++++++++++++++++ 5 files changed, 332 insertions(+), 12 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yaml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yaml create mode 100644 CONTRIBUTING.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml new file mode 100644 index 000000000..0ecb30c3c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -0,0 +1,106 @@ +name: Bug Report +description: Create a report to help us improve +title: "[BUG] " +labels: [bug] + +body: + - type: markdown + attributes: + value: | + ## Thank you for taking the time to report a bug + Please fill out this form as completely as possible. + + - type: input + id: what-happened + attributes: + label: What happened? + description: A clear and concise description of what the bug is. + placeholder: e.g., I get the following error when building a component with Viash. + validations: + required: true + + - type: input + id: reproduce + attributes: + label: Steps to reproduce + description: Steps to reproduce the behavior. + placeholder: | + Contents of `config.vsh.yaml`: + + ```yaml + ... + ``` + + Contents of `script.py`: + + ```python + ... + ``` + + Steps to reproduce: + + ```bash + ... + `` + validations: + required: true + + - type: input + id: expected-behavior + attributes: + label: Expected behavior + description: A clear and concise description of what you expected to happen. + placeholder: e.g., The application should save my data and remain stable. + validations: + required: true + + - type: textarea + id: logs + attributes: + label: Relevant log output + description: Please add any relevant logs or error messages. + placeholder: | + ``` + Your log output here. + ``` + + - type: input + id: version + attributes: + label: Version + description: The version of the software where the bug was encountered. + placeholder: | + - OS: [e.g., Ubuntu 20.04] + - Java Version: [e.g., 11] + - Viash Version: [e.g. 0.8.1] + - Python Version: [if applicable, e.g. 3.10] + - R Version: [if applicable, e.g. 4.0] + - Docker Version: [if applicable] + - Nextflow Version: [if applicable] + validations: + required: true + + - type: input + id: possible-solution + attributes: + label: Possible solution + description: Feel free to suggest a possible solution for the bug. + + + - type: checkboxes + id: confirm + attributes: + label: Confirmation + description: | + Please make sure you have checked the following. + options: + - label: I have searched the existing issues to make sure this is not a duplicate. + required: true + - label: I have provided clear and concise information about the bug. + required: true + + - type: input + id: additional-context + attributes: + label: Additional context + description: Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml new file mode 100644 index 000000000..8824e821d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -0,0 +1,70 @@ +name: Feature Request +description: Suggest an idea for this project +title: "[FEATURE] " +labels: [enhancement] + +body: + - type: markdown + attributes: + value: | + ## Thank you for your feature request + Please fill out this form to help us understand your idea. + + - type: input + id: feature-summary + attributes: + label: Feature summary + description: A brief summary of the feature. + placeholder: e.g., Add a dark mode option. + validations: + required: true + + - type: textarea + id: feature-description + attributes: + label: Feature description + description: A detailed description of the feature. + placeholder: Describe the feature you'd like to see, how it works, and what it would accomplish. + validations: + required: true + + - type: textarea + id: motivation + attributes: + label: Why is this feature beneficial? + description: Explain why this feature would be useful to the project and its users. + placeholder: e.g., A dark mode would reduce eye strain for users working in low-light conditions. + validations: + required: true + + - type: textarea + id: alternatives-considered + attributes: + label: Alternatives considered + description: Describe any alternative solutions or features you've considered. + placeholder: e.g., Instead of a full dark mode, a feature to adjust screen brightness could be implemented. + + + - type: input + id: possible-solution + attributes: + label: Possible solution + description: Feel free to suggest a possible implementation for the feature. + + - type: checkboxes + id: confirm + attributes: + label: Confirmation + description: | + Please confirm the following before submitting. + options: + - label: I have searched the existing issues to make sure this is not a duplicate request. + required: true + - label: I have provided clear and concise information about the feature. + required: true + + - type: markdown + attributes: + value: | + ### Thank you for contributing to our project! + Your feature request will be reviewed, and we'll get back to you as soon as possible. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index b1595154f..bbacd70ca 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,45 @@ ## Describe your changes -## Issue ticket number and link -Closes #xxxx (Replace xxxx with the GitHub issue number) + -## Checklist before requesting a review -- [ ] I have performed a self-review of my code +## Related issue(s) -- Does this PR contain: - - [ ] Breaking changes - - [ ] New functionality - - [ ] Major changes - - [ ] Minor changes - - [ ] Bug fixes + -- [ ] Proposed changes are described in the CHANGELOG.md +Closes #xxxx + +## Type of Change + +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] New functionality (non-breaking change which adds functionality) +- [ ] Major change (non-breaking change which modifies existing functionality) +- [ ] Minor change (non-breaking change which does not modify existing functionality) +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] This change requires a documentation update + +## Checklist + +Requirements: + +- [ ] I have read the [CONTRIBUTING](CONTRIBUTING.md) doc. +- [ ] I have performed a self-review of my code by checking the "Changed Files" tab. +- [ ] My code follows the code style of this project. + +Tests: + +- [ ] I have added tests that prove my fix is effective or that my feature works. +- [ ] New and existing unit tests pass locally with my changes. + +Documentation: + +- [ ] Proposed changes are described in the CHANGELOG.md. +- [ ] I have updated the documentation accordingly. + +## Test Environment + + -- [ ] Relevant unit tests have been added \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cc962e01..0ff28b28f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ TODO add summary * GitHub Actions: Test different versions of Nextflow (22.04.5, latest, and latest-edge) (PR #593). Testing the latest Edge version of Nextflow will allow us to catch notice changes in Nextflow earlier. +* Updates to the documentation and templates in the Git repo (#598, PR #600): + + - Add contributing guidelines. + + - Add issue templates. + + - Reworked the pull request template. + + # Viash 0.8.1 (2023-11-20): Minor bug fix to Nextflow workflows This release fixes a bug in the Nextflow platform where calling a workflow with the `.run()` function without specifying the `fromState` argument would result in an error when the input channel contained tuples with more than two elements. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..d8f023a33 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,108 @@ +# Contributing + +Welcome to the Viash project, and thank you for considering contributing! We are open to a variety of contributions, including documentation updates, bug fixes, new features, and more. By participating in this project, you agree to abide by our [Code of Conduct](CONDUCT.md). + +## Getting started + +* If you don't have one already, create a [GitHub account](https://github.com/signup/join). + +* Search for an existing issue for your problem or suggestion in the [issue tracker](https://github.com/viash-io/viash/issues). If you find a related issue, please comment there rather than creating a new issue. If none can be found, please [create a new issue](https://github.com/viash-io/viash/issues/new) for your problem or suggestion. Clearly describe your issue, including steps to reproduce when it is a bug, or some justification for a proposed improvement. + +## Requirements + +* [Java 11](https://www.oracle.com/java/technologies/javase-jdk11-downloads.html) or greater + +* [sbt](https://www.scala-sbt.org/) + +The following dependencies are required for passing some of the tests. + +* [Python 3.10](https://www.python.org/downloads/) or greater + +* [R 4.0](https://www.r-project.org/) or greater + +* [Nextflow](https://www.nextflow.io/) 22.04.5 or greater + +* [Docker](https://www.docker.com/) + +## Forking the code + +* [Fork](https://github.com/viash-io/viash/#fork-destination-box) the repository on GitHub to make a copy of the repository on your account. + +* Clone your fork to your local machine so you can edit the files. Make sure to check out the `develop` branch, since this is the branch where all development happens. + + ```bash + git clone https://github.com/your-username/viash.git + cd viash + git checkout develop + ``` + +* Create a new branch for your changes. It's best to keep your changes separate from the `develop` branch. + + ```bash + git branch -b your-branch-name + ``` + +## Making changes + +* Code standards: Ensure your code adheres to the [Scala style guide](https://docs.scala-lang.org/style/). + +* Write tests: If you add new features or fix bugs, write tests that cover your changes. Our project uses sbt for testing. + +* Run tests locally: Before submitting your changes, make sure all tests pass locally. Our GitHub Actions CI pipeline performs tests on different environments, but it's good practice to check everything beforehand. + + ```bash + sbt test + ``` + + If you couldn't install some of the required dependencies, you can skip the tests that require them. For example, to skip all tests requiring Docker and Nextflow: + + ```bash + sbt 'testOnly -- -l io.viash.DockerTest -l io.viash.NextflowTest' + ``` + +* Documentation: Update the documentation if your changes require it. This includes both in-code documentation and external documentation like README.md. + +## Submitting your changes + +* Update the changelog: Add a new entry to the [CHANGELOG.md](CHANGELOG.md) file that describes your changes. This entry should follow the following format: + + ```markdown + * `AffectedComponent`: A short description of the proposed changes (#issue, PR #pr, by @contributor). + Additional information to motivate the proposed changes, if need be. + ``` + +* Stage any changed or added files. + + ```sh + git add file1 file2 file3 + ``` + +* Commit your changes: Once you're happy with your changes, commit them to your branch. A commit message should consist of a short summary of the changes, followed by an empty line and a more detailed description. For trivial changes, the more detailed description can be omitted. + + ```sh + git commit + ``` + +* Push to your fork: Push your changes to your fork on GitHub. + + ```sh + git push origin your-branch-name + ``` + +* Create a pull pequest: Go to the viash-io/viash repository on GitHub and create a new pull request. Describe your changes and submit it for review. + +## Review process + +Our team will review your pull request. We might ask for changes or clarifications. Keep an eye on your pull request for feedback. + +## Additional information + +### Creating a build + +To create a build of Viash and install it in `~/.local/bin`, run the following commands: + +```bash +./configure --prefix=~/.local +make +make install +``` \ No newline at end of file From 7fcf5857f4cd6f454fef0585caf90341fccaae66 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Tue, 12 Dec 2023 09:09:56 +0100 Subject: [PATCH 4/9] Merge pull request #585 from viash-io/fix/non_strict_config (#603) * Merge pull request #585 from viash-io/fix/non_strict_config Fix config decoding not being enforcing strict mode * Fetch remote config from same branch as the branch being tested We may need to do this automatically in the future. --- CHANGELOG.md | 3 +++ src/main/scala/io/viash/config/package.scala | 3 +-- .../scala/io/viash/e2e/config_view/MainConfigViewSuite.scala | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ff28b28f..a0d07e368 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ TODO add summary - Reworked the pull request template. +## BUG FIXES + +* `config`: Fix the main level of a component config not enforcing strict mode and instead allowing any field to be specified (PR #585). # Viash 0.8.1 (2023-11-20): Minor bug fix to Nextflow workflows diff --git a/src/main/scala/io/viash/config/package.scala b/src/main/scala/io/viash/config/package.scala index 8b6ebab91..00d5f5de0 100644 --- a/src/main/scala/io/viash/config/package.scala +++ b/src/main/scala/io/viash/config/package.scala @@ -22,10 +22,9 @@ import io.circe.generic.extras.Configuration import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} package object config { + import io.viash.helpers.circe._ import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - implicit val customConfig: Configuration = Configuration.default.withDefaults - // encoders and decoders for Config implicit val encodeConfig: Encoder.AsObject[Config] = deriveConfiguredEncoder implicit val decodeConfig: Decoder[Config] = deriveConfiguredDecoderFullChecks diff --git a/src/test/scala/io/viash/e2e/config_view/MainConfigViewSuite.scala b/src/test/scala/io/viash/e2e/config_view/MainConfigViewSuite.scala index 88d4f3f94..8636501ad 100644 --- a/src/test/scala/io/viash/e2e/config_view/MainConfigViewSuite.scala +++ b/src/test/scala/io/viash/e2e/config_view/MainConfigViewSuite.scala @@ -25,7 +25,7 @@ class MainConfigViewSuite extends AnyFunSuite{ test("viash config view remote") { val stdout = TestHelper.testMain( "config", "view", - "https://raw.githubusercontent.com/viash-io/viash/develop/src/test/resources/testbash/config.vsh.yaml" + "https://raw.githubusercontent.com/viash-io/viash/develop_0_8/src/test/resources/testbash/config.vsh.yaml" ) assert(stdout.startsWith("functionality:")) From 6a3d96d55fec53a2fbe4f6c839efc7e0078f5b47 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Wed, 13 Dec 2023 11:24:30 +0100 Subject: [PATCH 5/9] Fix/dependency repositories without name (#607) * Don't require name to be filled in for repositories defined under dependencies Duplicate repository types to with and without the `name` field. Define a series of traits that connects the repositories with and without names * Fix some generalizations between repository traits and specialized constructor * Add changelog entry Update comments; we no longer have to anonymize repositories. * Remove copyRepoWithName as it isn't used --- CHANGELOG.md | 2 + .../exceptions/DependencyException.scala | 2 +- .../viash/functionality/Functionality.scala | 2 +- .../dependencies/AbstractGitRepository.scala | 3 +- .../dependencies/CopyableRepo.scala | 1 - .../dependencies/Dependency.scala | 2 +- .../dependencies/GitRepository.scala | 15 +--- .../dependencies/GitRepositoryTrait.scala | 29 ++++++++ .../dependencies/GitRepositoryWithName.scala | 68 +++++++++++++++++++ .../dependencies/GithubRepository.scala | 31 +-------- .../dependencies/GithubRepositoryTrait.scala | 48 +++++++++++++ .../GithubRepositoryWithName.scala | 66 ++++++++++++++++++ .../dependencies/LocalRepository.scala | 11 +-- .../dependencies/LocalRepositoryTrait.scala | 25 +++++++ .../LocalRepositoryWithName.scala | 55 +++++++++++++++ .../dependencies/Repository.scala | 21 ++---- .../dependencies/RepositoryWithName.scala | 31 +++++++++ .../dependencies/ViashhubRepository.scala | 32 ++------- .../ViashhubRepositoryTrait.scala | 47 +++++++++++++ .../ViashhubRepositoryWithName.scala | 67 ++++++++++++++++++ .../functionality/dependencies/package.scala | 42 ++++++++++++ .../io/viash/lenses/RepositoryLens.scala | 3 +- .../io/viash/schemas/CollectedSchemas.scala | 5 ++ 23 files changed, 511 insertions(+), 97 deletions(-) create mode 100644 src/main/scala/io/viash/functionality/dependencies/GitRepositoryTrait.scala create mode 100644 src/main/scala/io/viash/functionality/dependencies/GitRepositoryWithName.scala create mode 100644 src/main/scala/io/viash/functionality/dependencies/GithubRepositoryTrait.scala create mode 100644 src/main/scala/io/viash/functionality/dependencies/GithubRepositoryWithName.scala create mode 100644 src/main/scala/io/viash/functionality/dependencies/LocalRepositoryTrait.scala create mode 100644 src/main/scala/io/viash/functionality/dependencies/LocalRepositoryWithName.scala create mode 100644 src/main/scala/io/viash/functionality/dependencies/RepositoryWithName.scala create mode 100644 src/main/scala/io/viash/functionality/dependencies/ViashhubRepositoryTrait.scala create mode 100644 src/main/scala/io/viash/functionality/dependencies/ViashhubRepositoryWithName.scala diff --git a/CHANGELOG.md b/CHANGELOG.md index a0d07e368..a53a30635 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ TODO add summary * `config`: Fix the main level of a component config not enforcing strict mode and instead allowing any field to be specified (PR #585). +* `Repositories`: Fix a structural issue where a repository defined directly in a `dependency` would require the `name` field to be set (PR #607). Repository variants are created with and without the `name` field. Repositories under `.functionality.dependencies[]` use repositories without the `name` field, while repositories under `.functionality.repositories[]` use repositories with the `name` field. + # Viash 0.8.1 (2023-11-20): Minor bug fix to Nextflow workflows This release fixes a bug in the Nextflow platform where calling a workflow with the `.run()` function without specifying the `fromState` argument would result in an error when the input channel contained tuples with more than two elements. diff --git a/src/main/scala/io/viash/exceptions/DependencyException.scala b/src/main/scala/io/viash/exceptions/DependencyException.scala index 0fa84a6df..9b75457a0 100644 --- a/src/main/scala/io/viash/exceptions/DependencyException.scala +++ b/src/main/scala/io/viash/exceptions/DependencyException.scala @@ -28,7 +28,7 @@ case class MissingBuildYamlException(sourcePath: Path, dependency: Dependency) e } case class CheckoutException(repo: Repository) extends AbstractDependencyException { - override def getMessage(): String = s"Could not checkout remote repository ${repo.name} of type ${repo.`type`}" + override def getMessage(): String = s"Could not checkout remote repository of type ${repo.`type`}" } case class MissingDependencyException(dependencies: List[Dependency]) extends AbstractDependencyException { diff --git a/src/main/scala/io/viash/functionality/Functionality.scala b/src/main/scala/io/viash/functionality/Functionality.scala index 9047e70df..7f241e8bd 100644 --- a/src/main/scala/io/viash/functionality/Functionality.scala +++ b/src/main/scala/io/viash/functionality/Functionality.scala @@ -278,7 +278,7 @@ case class Functionality( |""".stripMargin, "yaml") @default("Empty") - repositories: List[Repository] = Nil, + repositories: List[RepositoryWithName] = Nil, // The variables below are for internal use and shouldn't be publicly documented // setting this to true will change the working directory diff --git a/src/main/scala/io/viash/functionality/dependencies/AbstractGitRepository.scala b/src/main/scala/io/viash/functionality/dependencies/AbstractGitRepository.scala index b5ec97bbf..0f381f007 100644 --- a/src/main/scala/io/viash/functionality/dependencies/AbstractGitRepository.scala +++ b/src/main/scala/io/viash/functionality/dependencies/AbstractGitRepository.scala @@ -23,12 +23,11 @@ import java.io.File import java.nio.file.Paths import io.viash.exceptions.CheckoutException -abstract class AbstractGitRepository extends Repository { +trait AbstractGitRepository extends Repository { val uri: String val storePath: String def copyRepo( - name: String, `type`: String, tag: Option[String], path: Option[String], diff --git a/src/main/scala/io/viash/functionality/dependencies/CopyableRepo.scala b/src/main/scala/io/viash/functionality/dependencies/CopyableRepo.scala index 23f5afd9a..4d69e9875 100644 --- a/src/main/scala/io/viash/functionality/dependencies/CopyableRepo.scala +++ b/src/main/scala/io/viash/functionality/dependencies/CopyableRepo.scala @@ -20,7 +20,6 @@ package io.viash.functionality.dependencies trait CopyableRepo[A <: CopyableRepo[A]] { self: A => def copyRepo( - name: String, `type`: String, tag: Option[String], path: Option[String], diff --git a/src/main/scala/io/viash/functionality/dependencies/Dependency.scala b/src/main/scala/io/viash/functionality/dependencies/Dependency.scala index 136deba39..93a110e60 100644 --- a/src/main/scala/io/viash/functionality/dependencies/Dependency.scala +++ b/src/main/scala/io/viash/functionality/dependencies/Dependency.scala @@ -121,7 +121,7 @@ case class Dependency( // Is this a dependency that will be built when `viash ns build` is run? def isLocalDependency: Boolean = workRepository.map{ - case r: LocalRepository => (r.path == None || r.path == Some(".")) && r.tag == None + case r: LocalRepositoryTrait => (r.path == None || r.path == Some(".")) && r.tag == None case _ => false }.getOrElse(false) } diff --git a/src/main/scala/io/viash/functionality/dependencies/GitRepository.scala b/src/main/scala/io/viash/functionality/dependencies/GitRepository.scala index cd888ca92..7c4c01081 100644 --- a/src/main/scala/io/viash/functionality/dependencies/GitRepository.scala +++ b/src/main/scala/io/viash/functionality/dependencies/GitRepository.scala @@ -32,8 +32,7 @@ import java.nio.file.Paths "yaml" ) @example( - """name: viash-testns - |type: git + """type: git |uri: git+https://gitlab.com/viash-io/viash.git |tag: 0.7.1 |path: src/test/resources/testns @@ -42,8 +41,6 @@ import java.nio.file.Paths ) @subclass("git") case class GitRepository( - name: String, - @description("Defines the repository as a Git repository.") `type`: String = "git", @@ -53,21 +50,15 @@ case class GitRepository( tag: Option[String], path: Option[String] = None, localPath: String = "" -) extends AbstractGitRepository { +) extends GitRepositoryTrait { def copyRepo( - name: String, `type`: String, tag: Option[String], path: Option[String], localPath: String ): GitRepository = { - copy(name, `type`, uri, tag, path, localPath) + copy(`type`, uri, tag, path, localPath) } - def getCheckoutUri(): String = uri - - // Strip the protocol and user credentials - val storePath = uri.replaceFirst("^.+://", "").replaceFirst(".+@", "") - } diff --git a/src/main/scala/io/viash/functionality/dependencies/GitRepositoryTrait.scala b/src/main/scala/io/viash/functionality/dependencies/GitRepositoryTrait.scala new file mode 100644 index 000000000..8cdfb2f0f --- /dev/null +++ b/src/main/scala/io/viash/functionality/dependencies/GitRepositoryTrait.scala @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.functionality.dependencies + +import io.viash.schemas._ + +trait GitRepositoryTrait extends AbstractGitRepository { + + def getCheckoutUri(): String = uri + + // Strip the protocol and user credentials + val storePath = uri.replaceFirst("^.+://", "").replaceFirst(".+@", "") + +} diff --git a/src/main/scala/io/viash/functionality/dependencies/GitRepositoryWithName.scala b/src/main/scala/io/viash/functionality/dependencies/GitRepositoryWithName.scala new file mode 100644 index 000000000..0d8120542 --- /dev/null +++ b/src/main/scala/io/viash/functionality/dependencies/GitRepositoryWithName.scala @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.functionality.dependencies + +import io.viash.schemas._ +import io.viash.helpers.IO +import io.viash.helpers.Exec +import java.io.File +import java.nio.file.Paths + +@description("A Git repository where remote dependency components can be found.") +@example( + """name: openpipeline + |type: git + |uri: git+https://github.com/openpipelines-bio/openpipeline.git + |tag: 0.8.0 + |""".stripMargin, + "yaml" +) +@example( + """name: viash-testns + |type: git + |uri: git+https://gitlab.com/viash-io/viash.git + |tag: 0.7.1 + |path: src/test/resources/testns + |""".stripMargin, + "yaml" + ) +@subclass("gitwithname") +case class GitRepositoryWithName( + name: String, + + @description("Defines the repository as a Git repository.") + `type`: String = "git", + + @description("The URI of the Git repository.") + @example("uri: \"git+https://github.com/openpipelines-bio/openpipeline.git\"", "yaml") + uri: String, + tag: Option[String], + path: Option[String] = None, + localPath: String = "" +) extends RepositoryWithName with GitRepositoryTrait { + + def copyRepo( + `type`: String, + tag: Option[String], + path: Option[String], + localPath: String + ): GitRepositoryWithName = { + copy("", `type`, uri, tag, path, localPath) + } + +} diff --git a/src/main/scala/io/viash/functionality/dependencies/GithubRepository.scala b/src/main/scala/io/viash/functionality/dependencies/GithubRepository.scala index 60c7d9d9c..94418d973 100644 --- a/src/main/scala/io/viash/functionality/dependencies/GithubRepository.scala +++ b/src/main/scala/io/viash/functionality/dependencies/GithubRepository.scala @@ -32,8 +32,7 @@ import java.nio.file.Paths "yaml" ) @example( - """name: viash-testns - |type: github + """type: github |repo: viash-io/viash |tag: 0.7.1 |path: src/test/resources/testns @@ -42,46 +41,22 @@ import java.nio.file.Paths ) @subclass("github") case class GithubRepository( - name: String, - @description("Defines the repository as a GitHub repository.") `type`: String = "github", - @description("The name of the GitHub repository.") - @example("repo: viash-io/viash", "yaml") repo: String, tag: Option[String], path: Option[String] = None, localPath: String = "" -) extends AbstractGitRepository { +) extends GithubRepositoryTrait { def copyRepo( - name: String, `type`: String, tag: Option[String], path: Option[String], localPath: String ): GithubRepository = { - copy(name, `type`, this.repo, tag, path, localPath) + copy(`type`, this.repo, tag, path, localPath) } - override def getCheckoutUri(): String = { - if (checkGitAuthentication(uri_nouser)) { - // First try https with bad user & password to disable asking credentials - // If successful, do checkout without the dummy credentials, don't want to store them in the repo remote address - uri - } else if (checkGitAuthentication(uri_ssh)) { - // Checkout with ssh key - uri_ssh - } else { - uri - } - } - - lazy val uri = s"https://github.com/$repo.git" - lazy val uri_ssh = s"git@github.com:$repo.git" - val fakeCredentials = "nouser:nopass@" // obfuscate the credentials a bit so we don't trigger GitGuardian - lazy val uri_nouser = s"https://${fakeCredentials}github.com/$repo.git" - - val storePath = repo // no need to add 'github.com' to the store path as 'type' (github) will be added } diff --git a/src/main/scala/io/viash/functionality/dependencies/GithubRepositoryTrait.scala b/src/main/scala/io/viash/functionality/dependencies/GithubRepositoryTrait.scala new file mode 100644 index 000000000..16f37cf19 --- /dev/null +++ b/src/main/scala/io/viash/functionality/dependencies/GithubRepositoryTrait.scala @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.functionality.dependencies + +import io.viash.schemas._ + +trait GithubRepositoryTrait extends AbstractGitRepository { + + @description("The name of the GitHub repository.") + @example("repo: viash-io/viash", "yaml") + val repo: String + + + override def getCheckoutUri(): String = { + if (checkGitAuthentication(uri_nouser)) { + // First try https with bad user & password to disable asking credentials + // If successful, do checkout without the dummy credentials, don't want to store them in the repo remote address + uri + } else if (checkGitAuthentication(uri_ssh)) { + // Checkout with ssh key + uri_ssh + } else { + uri + } + } + + lazy val uri = s"https://github.com/$repo.git" + lazy val uri_ssh = s"git@github.com:$repo.git" + val fakeCredentials = "nouser:nopass@" // obfuscate the credentials a bit so we don't trigger GitGuardian + lazy val uri_nouser = s"https://${fakeCredentials}github.com/$repo.git" + + val storePath = repo // no need to add 'github.com' to the store path as 'type' (github) will be added +} diff --git a/src/main/scala/io/viash/functionality/dependencies/GithubRepositoryWithName.scala b/src/main/scala/io/viash/functionality/dependencies/GithubRepositoryWithName.scala new file mode 100644 index 000000000..006817284 --- /dev/null +++ b/src/main/scala/io/viash/functionality/dependencies/GithubRepositoryWithName.scala @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.functionality.dependencies + +import io.viash.schemas._ +import io.viash.helpers.IO +import io.viash.helpers.Exec +import java.io.File +import java.nio.file.Paths + +@description("A GitHub repository where remote dependency components can be found.") +@example( + """name: openpipeline + |type: github + |repo: openpipelines-bio/openpipeline + |tag: 0.8.0 + |""".stripMargin, + "yaml" +) +@example( + """name: viash-testns + |type: github + |repo: viash-io/viash + |tag: 0.7.1 + |path: src/test/resources/testns + |""".stripMargin, + "yaml" + ) +@subclass("githubwithname") +case class GithubRepositoryWithName( + name: String, + + @description("Defines the repository as a GitHub repository.") + `type`: String = "github", + + repo: String, + tag: Option[String], + path: Option[String] = None, + localPath: String = "" +) extends RepositoryWithName with GithubRepositoryTrait { + + def copyRepo( + `type`: String, + tag: Option[String], + path: Option[String], + localPath: String + ): GithubRepositoryWithName = { + copy("", `type`, this.repo, tag, path, localPath) + } + +} diff --git a/src/main/scala/io/viash/functionality/dependencies/LocalRepository.scala b/src/main/scala/io/viash/functionality/dependencies/LocalRepository.scala index c90d10fbc..97635bcfa 100644 --- a/src/main/scala/io/viash/functionality/dependencies/LocalRepository.scala +++ b/src/main/scala/io/viash/functionality/dependencies/LocalRepository.scala @@ -27,8 +27,7 @@ import java.nio.file.Paths |""".stripMargin ) @exampleWithDescription( - """name: my_local_code - |type: local + """type: local |path: /additional_code/src |""".stripMargin, "yaml", @@ -36,24 +35,20 @@ import java.nio.file.Paths ) @subclass("local") case class LocalRepository( - name: String = "", - @description("Defines the repository as a locally present and available repository.") `type`: String = "local", tag: Option[String] = None, path: Option[String] = None, localPath: String = "" -) extends Repository { +) extends LocalRepositoryTrait { def copyRepo( - name: String, `type`: String, tag: Option[String], path: Option[String], localPath: String ): LocalRepository = { - copy(name, `type`, tag, path, localPath) + copy(`type`, tag, path, localPath) } - def subOutputPath: String = Paths.get(`type`, tag.getOrElse("")).toString() } diff --git a/src/main/scala/io/viash/functionality/dependencies/LocalRepositoryTrait.scala b/src/main/scala/io/viash/functionality/dependencies/LocalRepositoryTrait.scala new file mode 100644 index 000000000..7a5d5a275 --- /dev/null +++ b/src/main/scala/io/viash/functionality/dependencies/LocalRepositoryTrait.scala @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.functionality.dependencies +import io.viash.schemas._ +import java.nio.file.Paths + +trait LocalRepositoryTrait extends Repository { + + def subOutputPath: String = Paths.get(`type`, tag.getOrElse("")).toString(); +} diff --git a/src/main/scala/io/viash/functionality/dependencies/LocalRepositoryWithName.scala b/src/main/scala/io/viash/functionality/dependencies/LocalRepositoryWithName.scala new file mode 100644 index 000000000..a9461237d --- /dev/null +++ b/src/main/scala/io/viash/functionality/dependencies/LocalRepositoryWithName.scala @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.functionality.dependencies + +import io.viash.schemas._ +import java.nio.file.Paths + +@description( + """Defines a locally present and available repository. + |This can be used to define components from the same code base as the current component. + |Alternatively, this can be used to refer to a code repository present on the local hard-drive instead of fetchable remotely, for example during development. + |""".stripMargin +) +@exampleWithDescription( + """name: my_local_code + |type: local + |path: /additional_code/src + |""".stripMargin, + "yaml", + "Refer to a local code repository under `additional_code/src` referenced to the Viash Project Config file." +) +@subclass("localwithname") +case class LocalRepositoryWithName ( + name: String, + `type`: String = "local", + tag: Option[String] = None, + path: Option[String] = None, + localPath: String = "" +) extends RepositoryWithName with LocalRepositoryTrait { + + def copyRepo( + `type`: String, + tag: Option[String], + path: Option[String], + localPath: String + ): LocalRepositoryWithName = { + copy("", `type`, tag, path, localPath) + } + +} diff --git a/src/main/scala/io/viash/functionality/dependencies/Repository.scala b/src/main/scala/io/viash/functionality/dependencies/Repository.scala index 037c77dba..b38d99930 100644 --- a/src/main/scala/io/viash/functionality/dependencies/Repository.scala +++ b/src/main/scala/io/viash/functionality/dependencies/Repository.scala @@ -29,9 +29,6 @@ import java.nio.file.Files @subclass("GithubRepository") @subclass("ViashhubRepository") abstract class Repository extends CopyableRepo[Repository] { - @description("The identifier used to refer to this repository from dependencies.") - val name: String - @description("Defines the repository type. This determines how the repository will be fetched and handled.") val `type`: String @@ -46,7 +43,6 @@ abstract class Repository extends CopyableRepo[Repository] { val localPath: String def copyRepo( - name: String = this.name, `type`: String = this.`type`, tag: Option[String] = this.tag, path: Option[String] = this.path, @@ -67,24 +63,21 @@ object Repository { str match { case sugarSyntaxRegex("git+https", uri, tag) => Some(GitRepository( - "TODO generate name", uri = "https://" + uri, tag = getGitTag(tag) )) case sugarSyntaxRegex("github", repo, tag) => Some(GithubRepository( - "TODO generate name", repo = repo, tag = getGitTag(tag) )) case sugarSyntaxRegex("vsh", repo, tag) => Some(ViashhubRepository( - "TODO generate name", repo = repo, tag = getGitTag(tag) )) case sugarSyntaxRegex("local", repo, tag) => - Some(LocalRepository("TODO generate name")) + Some(LocalRepository()) case _ => None } } @@ -95,10 +88,8 @@ object Repository { // We just fetched a code base and we have to assume it will not change within this session. private val cachedRepos = scala.collection.mutable.ListBuffer[Repository]() private def getCachedRepository(repo: Repository): Option[Repository] = { - // We can't compare names because they don't hold actual information and can change between configs but still point to the same code base. - val anonymizedRepo = repo.copyRepo(name = "") - // Compare anonymized repos. Don't compare localPath as that is the information we're looking for. - val foundRepo = cachedRepos.find(p => p.copyRepo(localPath = "").equals(anonymizedRepo)) + // Don't compare localPath as that is the information we're looking for. + val foundRepo = cachedRepos.find(p => p.copyRepo(localPath = "").equals(repo)) // Map Some(foundRepo) to original repo but with localPath filled in, returns None if no cache found. foundRepo.map(r => repo.copyRepo(localPath = r.localPath)) } @@ -108,7 +99,7 @@ object Repository { case r: LocalRepository if r.path.isDefined && !r.path.get.startsWith("/") => // don't do anything, this repo is not reliably cacheable case _ => - cachedRepos.append(repo.copyRepo(name = "")) + cachedRepos.append(repo) } } @@ -127,12 +118,12 @@ object Repository { // Stopgap solution to be able to use built repositories which were not built with dependency aware Viash version. // TODO remove this section once it's deemed no longer necessary if (Paths.get(r3.localPath, "target").toFile().exists() && !Paths.get(r3.localPath, "target", ".build.yaml").toFile().exists()) { - Console.err.println(s"${Console.YELLOW}Creating temporary 'target/.build.yaml' file for ${r3.name} as this file seems to be missing.${Console.RESET}") + Console.err.println(s"${Console.YELLOW}Creating temporary 'target/.build.yaml' file for ${r3.`type`} as this file seems to be missing.${Console.RESET}") Files.createFile(Paths.get(r3.localPath, "target", ".build.yaml")) } r3 } - case r: LocalRepository if r.path.isDefined => { + case r: LocalRepositoryTrait if r.path.isDefined => { val localPath = r.path.get match { case s if s.startsWith("/") => // resolve path relative to the project root diff --git a/src/main/scala/io/viash/functionality/dependencies/RepositoryWithName.scala b/src/main/scala/io/viash/functionality/dependencies/RepositoryWithName.scala new file mode 100644 index 000000000..9470fbfc0 --- /dev/null +++ b/src/main/scala/io/viash/functionality/dependencies/RepositoryWithName.scala @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.functionality.dependencies + +import io.viash.schemas._ + +@description("Specifies a repository where dependency components can be found.") +@subclass("LocalRepositoryWithName") +@subclass("GitRepositoryWithName") +@subclass("GithubRepositoryWithName") +@subclass("ViashhubRepositoryWithName") +abstract class RepositoryWithName extends Repository { + @description("The identifier used to refer to this repository from dependencies.") + val name: String + +} diff --git a/src/main/scala/io/viash/functionality/dependencies/ViashhubRepository.scala b/src/main/scala/io/viash/functionality/dependencies/ViashhubRepository.scala index 445e6c955..7e0e88d7f 100644 --- a/src/main/scala/io/viash/functionality/dependencies/ViashhubRepository.scala +++ b/src/main/scala/io/viash/functionality/dependencies/ViashhubRepository.scala @@ -32,8 +32,7 @@ import java.nio.file.Paths "yaml" ) @example( - """name: viash-testns - |type: viashhub + """type: viashhub |repo: openpipelines-bio/openpipeline |tag: 0.7.1 |path: src/test/resources/testns @@ -42,46 +41,25 @@ import java.nio.file.Paths ) @subclass("viashhub") case class ViashhubRepository( - name: String, + // name: String, @description("Defines the repository as a Viash-Hub repository.") `type`: String = "vsh", - @description("The name of the Viash-Hub repository.") - @example("repo: openpipelines-bio/openpipeline", "yaml") repo: String, tag: Option[String], path: Option[String] = None, localPath: String = "" -) extends AbstractGitRepository { +) extends ViashhubRepositoryTrait { def copyRepo( - name: String, + // name: String, `type`: String, tag: Option[String], path: Option[String], localPath: String ): ViashhubRepository = { - copy(name, `type`, this.repo, tag, path, localPath) - } - - override def getCheckoutUri(): String = { - if (checkGitAuthentication(uri_nouser)) { - // First try https with bad user & password to disable asking credentials - // If successful, do checkout without the dummy credentials, don't want to store them in the repo remote address - uri - } else if (checkGitAuthentication(uri_ssh)) { - // Checkout with ssh key - uri_ssh - } else { - uri - } + copy(`type`, this.repo, tag, path, localPath) } - lazy val uri = s"https://viash-hub.com/$repo.git" - lazy val uri_ssh = s"git@viash-hub.com:$repo.git" - val fakeCredentials = "nouser:nopass@" // obfuscate the credentials a bit so we don't trigger GitGuardian - lazy val uri_nouser = s"https://${fakeCredentials}viash-hub.com/$repo.git" - - val storePath = repo // no need to add 'viash-hub.com' to the store path as 'type' (vsh) will be added } diff --git a/src/main/scala/io/viash/functionality/dependencies/ViashhubRepositoryTrait.scala b/src/main/scala/io/viash/functionality/dependencies/ViashhubRepositoryTrait.scala new file mode 100644 index 000000000..1273a1e53 --- /dev/null +++ b/src/main/scala/io/viash/functionality/dependencies/ViashhubRepositoryTrait.scala @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.functionality.dependencies + +import io.viash.schemas._ + +trait ViashhubRepositoryTrait extends AbstractGitRepository { + + @description("The name of the Viash-Hub repository.") + @example("repo: openpipelines-bio/openpipeline", "yaml") + val repo: String + + override def getCheckoutUri(): String = { + if (checkGitAuthentication(uri_nouser)) { + // First try https with bad user & password to disable asking credentials + // If successful, do checkout without the dummy credentials, don't want to store them in the repo remote address + uri + } else if (checkGitAuthentication(uri_ssh)) { + // Checkout with ssh key + uri_ssh + } else { + uri + } + } + + lazy val uri = s"https://viash-hub.com/$repo.git" + lazy val uri_ssh = s"git@viash-hub.com:$repo.git" + val fakeCredentials = "nouser:nopass@" // obfuscate the credentials a bit so we don't trigger GitGuardian + lazy val uri_nouser = s"https://${fakeCredentials}viash-hub.com/$repo.git" + + val storePath = repo // no need to add 'viash-hub.com' to the store path as 'type' (vsh) will be added +} diff --git a/src/main/scala/io/viash/functionality/dependencies/ViashhubRepositoryWithName.scala b/src/main/scala/io/viash/functionality/dependencies/ViashhubRepositoryWithName.scala new file mode 100644 index 000000000..ff6155831 --- /dev/null +++ b/src/main/scala/io/viash/functionality/dependencies/ViashhubRepositoryWithName.scala @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.functionality.dependencies + +import io.viash.schemas._ +import io.viash.helpers.IO +import io.viash.helpers.Exec +import java.io.File +import java.nio.file.Paths + +@description("A Viash-Hub repository where remote dependency components can be found.") +@example( + """name: openpipeline + |type: viashhub + |repo: openpipelines-bio/openpipeline + |tag: 0.8.0 + |""".stripMargin, + "yaml" +) +@example( + """name: viash-testns + |type: viashhub + |repo: openpipelines-bio/openpipeline + |tag: 0.7.1 + |path: src/test/resources/testns + |""".stripMargin, + "yaml" + ) +@subclass("viashhubwithname") +case class ViashhubRepositoryWithName( + name: String, + + @description("Defines the repository as a Viash-Hub repository.") + `type`: String = "vsh", + + repo: String, + tag: Option[String], + path: Option[String] = None, + localPath: String = "" +) extends RepositoryWithName with ViashhubRepositoryTrait { + + def copyRepo( + // name: String, + `type`: String, + tag: Option[String], + path: Option[String], + localPath: String + ): ViashhubRepositoryWithName = { + copy("", `type`, this.repo, tag, path, localPath) + } + +} diff --git a/src/main/scala/io/viash/functionality/dependencies/package.scala b/src/main/scala/io/viash/functionality/dependencies/package.scala index 6d0e4838f..42bae48b8 100644 --- a/src/main/scala/io/viash/functionality/dependencies/package.scala +++ b/src/main/scala/io/viash/functionality/dependencies/package.scala @@ -21,6 +21,7 @@ import io.circe.{Decoder, Encoder, Json} import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ import cats.syntax.functor._ +import dependencies.GithubRepository package object dependencies { @@ -32,6 +33,7 @@ package object dependencies { implicit val encodeGithubRepository: Encoder.AsObject[GithubRepository] = deriveConfiguredEncoder implicit val encodeViashhubRepository: Encoder.AsObject[ViashhubRepository] = deriveConfiguredEncoder implicit val encodeLocalRepository: Encoder.AsObject[LocalRepository] = deriveConfiguredEncoder + // Repositories _WithName are also of type Repository, so we must define an encoder for them as well implicit def encodeRepository[A <: Repository]: Encoder[A] = Encoder.instance { par => val typeJson = Json.obj("type" -> Json.fromString(par.`type`)) @@ -40,6 +42,26 @@ package object dependencies { case s: GithubRepository => encodeGithubRepository(s) case s: ViashhubRepository => encodeViashhubRepository(s) case s: LocalRepository => encodeLocalRepository(s) + case s: GitRepositoryWithName => encodeGitRepositoryWithName(s) + case s: GithubRepositoryWithName => encodeGithubRepositoryWithName(s) + case s: ViashhubRepositoryWithName => encodeViashhubRepositoryWithName(s) + case s: LocalRepositoryWithName => encodeLocalRepositoryWithName(s) + } + objJson deepMerge typeJson + } + + implicit val encodeGitRepositoryWithName: Encoder.AsObject[GitRepositoryWithName] = deriveConfiguredEncoder + implicit val encodeGithubRepositoryWithName: Encoder.AsObject[GithubRepositoryWithName] = deriveConfiguredEncoder + implicit val encodeViashhubRepositoryWithName: Encoder.AsObject[ViashhubRepositoryWithName] = deriveConfiguredEncoder + implicit val encodeLocalRepositoryWithName: Encoder.AsObject[LocalRepositoryWithName] = deriveConfiguredEncoder + implicit def encodeRepositoryWithName[A <: RepositoryWithName]: Encoder[A] = Encoder.instance { + par => + val typeJson = Json.obj("type" -> Json.fromString(par.`type`)) + val objJson = par match { + case s: GitRepositoryWithName => encodeGitRepositoryWithName(s) + case s: GithubRepositoryWithName => encodeGithubRepositoryWithName(s) + case s: ViashhubRepositoryWithName => encodeViashhubRepositoryWithName(s) + case s: LocalRepositoryWithName => encodeLocalRepositoryWithName(s) } objJson deepMerge typeJson } @@ -66,4 +88,24 @@ package object dependencies { decoder(cursor) } + implicit val decodeGitRepositoryWithName: Decoder[GitRepositoryWithName] = deriveConfiguredDecoderFullChecks + implicit val decodeGithubRepositoryWithName: Decoder[GithubRepositoryWithName] = deriveConfiguredDecoderFullChecks + implicit val decodeViashhubRepositoryWithName: Decoder[ViashhubRepositoryWithName] = deriveConfiguredDecoderFullChecks + implicit val decodeLocalRepositoryWithName: Decoder[LocalRepositoryWithName] = deriveConfiguredDecoderFullChecks + implicit def decodeRepositoryWithName: Decoder[RepositoryWithName] = Decoder.instance { + cursor => + val decoder: Decoder[RepositoryWithName] = + cursor.downField("type").as[String] match { + case Right("git") => decodeGitRepositoryWithName.widen + case Right("github") => decodeGithubRepositoryWithName.widen + case Right("vsh") => decodeViashhubRepositoryWithName.widen + case Right("local") => decodeLocalRepositoryWithName.widen + case Right(typ) => + DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[LocalRepositoryWithName](typ, List("git", "github", "vsh", "local")).widen + case Left(exception) => throw exception + } + + decoder(cursor) + } + } \ No newline at end of file diff --git a/src/main/scala/io/viash/lenses/RepositoryLens.scala b/src/main/scala/io/viash/lenses/RepositoryLens.scala index b199ed054..642e73c87 100644 --- a/src/main/scala/io/viash/lenses/RepositoryLens.scala +++ b/src/main/scala/io/viash/lenses/RepositoryLens.scala @@ -19,9 +19,10 @@ package io.viash.lenses import monocle.PLens import io.viash.functionality.dependencies.Repository +import io.viash.functionality.dependencies.RepositoryWithName object RepositoryLens { - val nameLens = PLens[Repository, Repository, String, String](r => r.name)(s => r => r.copyRepo(name = s)) + // val nameLens = PLens[RepositoryWithName, RepositoryWithName, String, String](r => r.name)(s => r => r.copyRepo(name = s)) val typeLens = PLens[Repository, Repository, String, String](r => r.`type`)(s => r => r.copyRepo(`type` = s)) val tagLens = PLens[Repository, Repository, Option[String], Option[String]](r => r.tag)(s => r => r.copyRepo(tag = s)) val pathLens = PLens[Repository, Repository, Option[String], Option[String]](r => r.path)(s => r => r.copyRepo(path = s)) diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala index 879bb9120..0d847fe9a 100644 --- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala +++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala @@ -167,6 +167,11 @@ object CollectedSchemas { getMembers[GitRepository](), getMembers[GithubRepository](), getMembers[ViashhubRepository](), + getMembers[RepositoryWithName](), + getMembers[LocalRepositoryWithName](), + getMembers[GitRepositoryWithName](), + getMembers[GithubRepositoryWithName](), + getMembers[ViashhubRepositoryWithName](), ) private def trimTypeName(s: String) = { From 9a13dd8cd27253bf29d981547596e74f914b1ce8 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Wed, 13 Dec 2023 11:54:27 +0100 Subject: [PATCH 6/9] use "local" to match the repository name instead of using that in the sugar syntax regex (#609) * use "local" to match the repository name instead of using that in the sugar syntax regex regex was needed to extract repo and tag names, but this is irrelevant here. It would have required something like `local://...` notation. * Add changelog entry * Add sugar syntax notation for local repositories, but now properly * Add test for the new sugar syntax option --- CHANGELOG.md | 3 +++ .../io/viash/functionality/dependencies/Repository.scala | 7 ++++++- .../resources/testnextflowvdsl3/src/wf/config.vsh.yaml | 2 +- .../io/viash/functionality/dependencies/Repository.scala | 6 ++++++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a53a30635..6c6985099 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ TODO add summary * `config`: Fix the main level of a component config not enforcing strict mode and instead allowing any field to be specified (PR #585). + +* `dependencies`: Allow the user to define a local dependency with specifying `repository: local` as sugar syntax (PR #609). A local repository is the default value so it's not required to be filled in, but allowing it with a sensible sugar syntax makes sense. + * `Repositories`: Fix a structural issue where a repository defined directly in a `dependency` would require the `name` field to be set (PR #607). Repository variants are created with and without the `name` field. Repositories under `.functionality.dependencies[]` use repositories without the `name` field, while repositories under `.functionality.repositories[]` use repositories with the `name` field. # Viash 0.8.1 (2023-11-20): Minor bug fix to Nextflow workflows diff --git a/src/main/scala/io/viash/functionality/dependencies/Repository.scala b/src/main/scala/io/viash/functionality/dependencies/Repository.scala index b38d99930..621e61567 100644 --- a/src/main/scala/io/viash/functionality/dependencies/Repository.scala +++ b/src/main/scala/io/viash/functionality/dependencies/Repository.scala @@ -76,7 +76,12 @@ object Repository { repo = repo, tag = getGitTag(tag) )) - case sugarSyntaxRegex("local", repo, tag) => + case sugarSyntaxRegex("local", path, tag) => + Some(LocalRepository( + path = Some(path), + tag = getGitTag(tag) + )) + case "local" => Some(LocalRepository()) case _ => None } diff --git a/src/test/resources/testnextflowvdsl3/src/wf/config.vsh.yaml b/src/test/resources/testnextflowvdsl3/src/wf/config.vsh.yaml index 96a5e44c7..ef034682d 100644 --- a/src/test/resources/testnextflowvdsl3/src/wf/config.vsh.yaml +++ b/src/test/resources/testnextflowvdsl3/src/wf/config.vsh.yaml @@ -30,7 +30,7 @@ functionality: dependencies: - name: step1 # local is implicit - name: step2 - # repository: local # TODO: this should also work + repository: local - name: step3 repository: mylocal repositories: diff --git a/src/test/scala/io/viash/functionality/dependencies/Repository.scala b/src/test/scala/io/viash/functionality/dependencies/Repository.scala index 781db896c..631469a65 100644 --- a/src/test/scala/io/viash/functionality/dependencies/Repository.scala +++ b/src/test/scala/io/viash/functionality/dependencies/Repository.scala @@ -36,6 +36,12 @@ class RepositoryTest extends AnyFunSuite { assert(repo.get.isInstanceOf[LocalRepository]) } + test("Repository.unapply: handles local dependency syntax") { + val repo = Repository.unapply("local") + assert(repo.isDefined) + assert(repo.get.isInstanceOf[LocalRepository]) + } + test("Repository.unapply: returns None for unrecognized syntax") { val repo = Repository.unapply("unknown://foo.bar") assert(repo.isEmpty) From a379de0228efaf46df522280dd414b89f8ba7f81 Mon Sep 17 00:00:00 2001 From: Dries Schaumont <5946712+DriesSchaumont@users.noreply.github.com> Date: Wed, 13 Dec 2023 15:47:00 +0100 Subject: [PATCH 7/9] NextflowPlatform: Fix resolving relative paths wrt the param_list file (#592) * Do not resolve remote paths relative to the --param_list file * alternative implementation for resolving the sibling * implement function using regex instead of nxf code * Apply suggestions from code review * check for nested paths * fix helper code * Add function tests * Run tests only when nextflow installed * Revert CHANGELOG changes * move groovy tests to be run by scala * update * normalize path in nxf test * try resolve issues between java 11 and higher with regex * fix regex * fix group * more options * another attempt at fixing the test for java 11 * contemplating an unorthodox spatial realignment of a flat-topped piece of furniture --------- Co-authored-by: Robrecht Cannoodt --- CHANGELOG.md | 4 + .../nextflow/channel/_parseParamList.nf | 7 +- .../functions/_resolveSiblingIfNotAbsolute.nf | 18 +++++ .../functions/_stringIsAbsolutePath.nf | 16 ++++ .../testResolveSiblingIfNotAbsolute.nf | 76 +++++++++++++++++++ .../NextflowHelperFunctionTest.scala | 58 ++++++++++++++ 6 files changed, 176 insertions(+), 3 deletions(-) create mode 100644 src/main/resources/io/viash/platforms/nextflow/functions/_resolveSiblingIfNotAbsolute.nf create mode 100644 src/main/resources/io/viash/platforms/nextflow/functions/_stringIsAbsolutePath.nf create mode 100644 src/test/resources/platforms/nextflow/functions/testResolveSiblingIfNotAbsolute.nf create mode 100644 src/test/scala/io/viash/platforms/nextflow/functions/NextflowHelperFunctionTest.scala diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c6985099..ad1bd34f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ TODO add summary +## BUG FIXES + +`NextflowPlatform`: Do not resolve remote paths relative to the --param_list file (PR #592). + ## MINOR CHANGES * `NextflowTestHelper`: Do not hardcode a version of Nextflow in the testbench, diff --git a/src/main/resources/io/viash/platforms/nextflow/channel/_parseParamList.nf b/src/main/resources/io/viash/platforms/nextflow/channel/_parseParamList.nf index 741682b6c..b02a50132 100644 --- a/src/main/resources/io/viash/platforms/nextflow/channel/_parseParamList.nf +++ b/src/main/resources/io/viash/platforms/nextflow/channel/_parseParamList.nf @@ -90,11 +90,12 @@ def _parseParamList(param_list, Map config) { def par = config.functionality.allArguments.find{it.plainName == parName} if (par && par.type == "file" && par.direction == "input") { if (parValue instanceof Collection) { - parValue = parValue.collect{path -> - path instanceof String ? paramListPath.resolveSibling(path) : path + parValue = parValue.collectMany{path -> + def x = _resolveSiblingIfNotAbsolute(path, paramListPath) + x instanceof Collection ? x : [x] } } else { - parValue = parValue instanceof String ? paramListPath.resolveSibling(parValue) : parValue + parValue = _resolveSiblingIfNotAbsolute(parValue, paramListPath) } } [parName, parValue] diff --git a/src/main/resources/io/viash/platforms/nextflow/functions/_resolveSiblingIfNotAbsolute.nf b/src/main/resources/io/viash/platforms/nextflow/functions/_resolveSiblingIfNotAbsolute.nf new file mode 100644 index 000000000..ec007f12d --- /dev/null +++ b/src/main/resources/io/viash/platforms/nextflow/functions/_resolveSiblingIfNotAbsolute.nf @@ -0,0 +1,18 @@ +/** + * Resolve a path relative to the current file. + * + * @param str The path to resolve, as a String. + * @param parentPath The path to resolve relative to, as a Path. + * + * @return The path that may have been resovled, as a Path. + */ +def _resolveSiblingIfNotAbsolute(str, parentPath) { + if (str !instanceof String) { + return str + } + if (!_stringIsAbsolutePath(str)) { + return parentPath.resolveSibling(str) + } else { + return file(str, hidden: true) + } +} diff --git a/src/main/resources/io/viash/platforms/nextflow/functions/_stringIsAbsolutePath.nf b/src/main/resources/io/viash/platforms/nextflow/functions/_stringIsAbsolutePath.nf new file mode 100644 index 000000000..d3a339561 --- /dev/null +++ b/src/main/resources/io/viash/platforms/nextflow/functions/_stringIsAbsolutePath.nf @@ -0,0 +1,16 @@ +/** + * Check whether a path as a string is absolute. + * + * In the past, we tried using `file(., relative: true).isAbsolute()`, + * but the 'relative' option was added in 22.10.0. + * + * @param path The path to check, as a String. + * + * @return Whether the path is absolute, as a boolean. + */ +def _stringIsAbsolutePath(path) { + _resolve_URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*:)?\\/.+/ + + assert path instanceof String + return _resolve_URL_PROTOCOL.matcher(path).matches() +} \ No newline at end of file diff --git a/src/test/resources/platforms/nextflow/functions/testResolveSiblingIfNotAbsolute.nf b/src/test/resources/platforms/nextflow/functions/testResolveSiblingIfNotAbsolute.nf new file mode 100644 index 000000000..12d46b184 --- /dev/null +++ b/src/test/resources/platforms/nextflow/functions/testResolveSiblingIfNotAbsolute.nf @@ -0,0 +1,76 @@ + +include { _resolveSiblingIfNotAbsolute } from params.workflowHelper + +// wherever optional groups are used, this is due to java 11 behaving a little +// bit differently than java 17 and 21. However, both behaviours should +// produce paths that are equally valid. + +// for compatibility with java 11 +def pwd = java.nio.file.Paths.get(".").toAbsolutePath().normalize() + +def testCases = [ + "/absolute/another/param_list.yaml": [ + "relative/path": ["/absolute/another/relative/path", "UnixPath"], + "/absolute/path": ["/absolute/path", "UnixPath"], + "s3://bucket/path": ["/bucket/path", "S3Path"], + "../foo/bar": ["/absolute(/another/..)?/foo/bar", "UnixPath"], + "file:///absolute/path": ["/absolute/path", "UnixPath"] + ], + "/absolute/../param_list.yaml": [ + "relative/path": ["(/absolute/..)?/relative/path", "UnixPath"], + "/absolute/path": ["/absolute/path", "UnixPath"], + "s3://bucket/path": ["/bucket/path", "S3Path"], + "../foo/bar": ["((/absolute/..)?/..)?/foo/bar", "UnixPath"], + "file:///absolute/path": ["/absolute/path", "UnixPath"] + ], + "relative/param_list.yaml": [ + "relative/path": ["($pwd/)?relative/relative/path", "UnixPath"], + "/absolute/path": ["/absolute/path", "UnixPath"], + "s3://bucket/path": ["/bucket/path", "S3Path"], + "../foo/bar": ["($pwd/)?(relative/../)?foo/bar", "UnixPath"], + "file:///absolute/path": ["/absolute/path", "UnixPath"] + ], + "s3://s3bucket/param_list.yaml": [ + "relative/path": ["/s3bucket/relative/path", "S3Path"], + "/absolute/path": ["/absolute/path", "UnixPath"], + "s3://bucket/path": ["/bucket/path", "S3Path"], + "../foo/bar": ["/s3bucket/(../)?foo/bar", "S3Path"], + "file:///absolute/path": ["/absolute/path", "UnixPath"] + ], + "https://remote_url/extra_dir/param_list.yaml": [ + // "relative/path": ["remote_url/extra_dir/relative/path", "XPath"], + "/absolute/path": ["/absolute/path", "UnixPath"], + "s3://bucket/path": ["/bucket/path", "S3Path"], + // "../foo/bar": ["/remote_url/extra_dir/../foo/bar", "XPath"], + "file:///absolute/path": ["/absolute/path", "UnixPath"] + ], + "/param_list.yaml": [ + "relative/path": ["/relative/path", "UnixPath"], + "/absolute/path": ["/absolute/path", "UnixPath"], + "s3://bucket/path": ["/bucket/path", "S3Path"], + "../foo/bar": ["/(../)?foo/bar", "UnixPath"], + "file:///absolute/path": ["/absolute/path", "UnixPath"] + ], + "/absolute/param_list.yaml": [ + "relative/path": ["/absolute/relative/path", "UnixPath"], + "/absolute/path": ["/absolute/path", "UnixPath"], + "s3://bucket/path": ["/bucket/path", "S3Path"], + "../foo/bar": ["(/absolute/..)?/foo/bar", "UnixPath"], + "file:///absolute/path": ["/absolute/path", "UnixPath"] + ] + +] + + +testCases.each { parentPath, providerTestCases -> + providerTestCases.each { path, expected -> + def expectedLocation = expected[0] + def expectedClass = expected[1] + def parentPathObject = file(parentPath, relative: true, hidden: true) + def resolvedPath = _resolveSiblingIfNotAbsolute(path, parentPathObject) + println("_resolveSiblingIfNotAbsolute(\"${parentPath}\", \"${path}\"): ${resolvedPath}, .getClass(): ${resolvedPath.getClass()}") + assert resolvedPath.toString() ==~ expectedLocation + assert resolvedPath.getClass().getSimpleName() == expectedClass + } + println("") +} \ No newline at end of file diff --git a/src/test/scala/io/viash/platforms/nextflow/functions/NextflowHelperFunctionTest.scala b/src/test/scala/io/viash/platforms/nextflow/functions/NextflowHelperFunctionTest.scala new file mode 100644 index 000000000..dc93e9925 --- /dev/null +++ b/src/test/scala/io/viash/platforms/nextflow/functions/NextflowHelperFunctionTest.scala @@ -0,0 +1,58 @@ +package io.viash.platforms.nextflow + +import io.viash.helpers.{IO, Logger} +import io.viash.{DockerTest, NextflowTest, TestHelper} +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite + +import java.io.File +import java.nio.file.{Files, Path, Paths} +import scala.jdk.CollectionConverters._ + +import NextflowTestHelper._ + +class NextflowHelperFunctionTest extends AnyFunSuite with BeforeAndAfterAll { + + Logger.UseColorOverride.value = Some(false) + // temporary folder to work in + private val temporaryFolder = IO.makeTemp("nextflow_helper_function_test") + + // some paths + private val testDir = temporaryFolder.resolve("test_resources") + private val workflowHelper = temporaryFolder.resolve("WorkflowHelper.nf") + + // path to namespace components + private val nextflowDir = Paths.get(getClass.getResource("/platforms/nextflow/").getPath) + + // copy resources to temporary folder so we can test in a clean environment + IO.copyFolder(nextflowDir, testDir) + IO.write(NextflowHelper.workflowHelper.toString, workflowHelper) + + // recursively search for files starting with 'test' and ending with '.nf' in 'functionDir: Path' + // and return a list of paths to those files + private val testFiles = Files.walk(testDir) + .iterator + .asScala + .filter(Files.isRegularFile(_)) + .filter(_.getFileName.toString.matches("^test.*\\.nf$")) + + // run all tests + for (testFile <- testFiles) { + test(s"Testing ${testFile}", NextflowTest) { + val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( + mainScript = testFile.toString, + args = List( + "--workflowHelper", workflowHelper.toString + ), + cwd = temporaryFolder.toFile() + ) + + assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") + } + } + + // delete temp files + override def afterAll(): Unit = { + IO.deleteRecursively(temporaryFolder) + } +} From 089a36510c211b8a923f24b070f2745192f7dd1b Mon Sep 17 00:00:00 2001 From: Dries Schaumont <5946712+DriesSchaumont@users.noreply.github.com> Date: Thu, 14 Dec 2023 11:30:55 +0100 Subject: [PATCH 8/9] NextflowPlatform: fix finding .build.yaml when worktree contains a symlink directory (#611) * NextflowPlatform: fix finding .build.yaml when worktree contains symlink directory * NextflowPlatform: fix finding .build.yaml when worktree contains symlink directory * Add missing 'def' * Add CHANGELOG entry * Implement test --- CHANGELOG.md | 2 ++ .../nextflow/functions/getRootDir.nf | 3 ++- .../dependencies/Dependency.scala | 3 ++- .../nextflow/NextflowScriptTest.scala | 19 +++++++++++++++++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad1bd34f2..e2a74252e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ TODO add summary `NextflowPlatform`: Do not resolve remote paths relative to the --param_list file (PR #592). +`NextflowPlatform`: Allow finding `.build.yaml` file when worktree contains a directory that is a symlink (PR #611). + ## MINOR CHANGES * `NextflowTestHelper`: Do not hardcode a version of Nextflow in the testbench, diff --git a/src/main/resources/io/viash/platforms/nextflow/functions/getRootDir.nf b/src/main/resources/io/viash/platforms/nextflow/functions/getRootDir.nf index 4700810d0..815f21d5a 100644 --- a/src/main/resources/io/viash/platforms/nextflow/functions/getRootDir.nf +++ b/src/main/resources/io/viash/platforms/nextflow/functions/getRootDir.nf @@ -1,6 +1,7 @@ // Recurse upwards until we find a '.build.yaml' file -def _findBuildYamlFile(path) { +def _findBuildYamlFile(pathPossiblySymlink) { + def path = pathPossiblySymlink.toRealPath() def child = path.resolve(".build.yaml") if (java.nio.file.Files.isDirectory(path) && java.nio.file.Files.exists(child)) { return child diff --git a/src/main/scala/io/viash/functionality/dependencies/Dependency.scala b/src/main/scala/io/viash/functionality/dependencies/Dependency.scala index 93a110e60..7ca895e27 100644 --- a/src/main/scala/io/viash/functionality/dependencies/Dependency.scala +++ b/src/main/scala/io/viash/functionality/dependencies/Dependency.scala @@ -167,7 +167,8 @@ object Dependency { } // Traverse the folder upwards until a `.build.yaml` is found but do not traverse beyond `repoPath`. - def findBuildYamlFile(path: Path, repoPath: Path): Option[Path] = { + def findBuildYamlFile(pathPossiblySymlink: Path, repoPath: Path): Option[Path] = { + val path = pathPossiblySymlink.toRealPath() val child = path.resolve(".build.yaml") if (Files.isDirectory(path) && Files.exists(child)) { Some(child) diff --git a/src/test/scala/io/viash/platforms/nextflow/NextflowScriptTest.scala b/src/test/scala/io/viash/platforms/nextflow/NextflowScriptTest.scala index 2dccf6593..9f42e52ed 100644 --- a/src/test/scala/io/viash/platforms/nextflow/NextflowScriptTest.scala +++ b/src/test/scala/io/viash/platforms/nextflow/NextflowScriptTest.scala @@ -61,6 +61,25 @@ class NextflowScriptTest extends AnyFunSuite with BeforeAndAfterAll { assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") } + test("Run config pipeline with symlink", NextflowTest) { + val newWorkflowPath = Paths.get(tempFolStr, "workflowsAsSymlink") + Files.createDirectories(newWorkflowPath) + val symlinkFolder = Paths.get(newWorkflowPath.toString(), "workflow") + Files.createSymbolicLink(symlinkFolder, Paths.get(tempFolStr, "target/nextflow/wf/")) + val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( + mainScript = "workflowsAsSymlink/workflow/main.nf", + args = List( + "--id", "foo", + "--input1", "resources/lines*.txt", + "--input2", "resources/lines3.txt", + "--publish_dir", "output" + ), + cwd = tempFolFile + ) + + assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") + } + // TODO: use TestHelper.testMainWithStdErr instead of NextflowTestHelper.run; i.e. viash test test("Test workflow", DockerTest, NextflowTest) { val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( From bad11390ebcf444916bc03635dc8f8d3cae0c661 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 14 Dec 2023 15:18:58 +0100 Subject: [PATCH 9/9] Prepare release of Viash 0.8.2 --- CHANGELOG.md | 15 ++++++--------- build.sbt | 2 +- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2a74252e..4609a5842 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,6 @@ -# Viash 0.x.x (yyyy-MM-dd): TODO Add title +# Viash 0.8.2 (2023-12-14): Minor changes and bug fixes -TODO add summary - -## BUG FIXES - -`NextflowPlatform`: Do not resolve remote paths relative to the --param_list file (PR #592). - -`NextflowPlatform`: Allow finding `.build.yaml` file when worktree contains a directory that is a symlink (PR #611). +This release fixes a few bugs regarding dependencies and how the Nextflow platform handles Paths. ## MINOR CHANGES @@ -28,11 +22,14 @@ TODO add summary * `config`: Fix the main level of a component config not enforcing strict mode and instead allowing any field to be specified (PR #585). - * `dependencies`: Allow the user to define a local dependency with specifying `repository: local` as sugar syntax (PR #609). A local repository is the default value so it's not required to be filled in, but allowing it with a sensible sugar syntax makes sense. * `Repositories`: Fix a structural issue where a repository defined directly in a `dependency` would require the `name` field to be set (PR #607). Repository variants are created with and without the `name` field. Repositories under `.functionality.dependencies[]` use repositories without the `name` field, while repositories under `.functionality.repositories[]` use repositories with the `name` field. +* `NextflowPlatform`: Do not resolve remote paths relative to the --param_list file (PR #592). + +* `NextflowPlatform`: Allow finding `.build.yaml` file when worktree contains a directory that is a symlink (PR #611). + # Viash 0.8.1 (2023-11-20): Minor bug fix to Nextflow workflows This release fixes a bug in the Nextflow platform where calling a workflow with the `.run()` function without specifying the `fromState` argument would result in an error when the input channel contained tuples with more than two elements. diff --git a/build.sbt b/build.sbt index a09187698..460a8de3d 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ name := "viash" -version := "0.8.2dev" +version := "0.8.2" scalaVersion := "2.13.10"