Dyn uri test

CHANGELOG.md

@@ -1,6 +1,18 @@
 # Changelog
 
-## [0.8.7] 2024-10-23
+## [0.9.0] TBD
+
+- Shift metadata and configuration to `quilt` scope of nextflow.config
+- Moved test workflows into `wf` folder
+- Write the output URI when publishing
+- Improve handling of dynamically-specified URIs
+- Rewrite README.md, splitting out developer documentation to README_DEV.md
+
+## [0.8.8] 2024-10-31 UNPUBLISHED
+
+- Debug build
+
+## [0.8.7] 2024-10-23 UNPUBLISHED
 
 - Use package cache instead of `params` to find output URIs
   (in order to support dynamic URIs set by, e.g. `main.nf`)

Makefile

@@ -4,7 +4,6 @@ WRITE_BUCKET ?= udp-spec
 FRAGMENT ?= &path=.
 NF_DIR ?= ../nextflow
 NF_BIN ?= ./launch.sh
-PATH_NF ?= ./main.path.nf
 PID ?= $$$$
 PIPELINE ?= sarek
 QUERY ?= ?Name=$(USER)&Owner=Kevin+Moore&Date=2023-03-07&Type=CRISPR&Notebook+URL=http%3A%2F%2Fexample.com
@@ -73,6 +72,9 @@ pkg-test: compile #-all
 	echo "$(TEST_URI)"
 	$(NF_BIN) run ./main.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)"
 
+dyn-test: compile #-all
+	$(NF_BIN) run wf/main.dynamic.nf -profile standard -plugins $(PROJECT)
+
 s3-overlay: compile
 	$(NF_BIN) run ./main.nf --plugins $(PROJECT) --outdir "$(S3_BASE)/s3-overlay" --input "$(S3_BASE)/s3-in"  
 
@@ -87,13 +89,13 @@ s3-out: compile
 
 pkg-fail: compile
 	echo "$(TEST_URI)"
-	$(NF_BIN) run ./fail.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)"
+	$(NF_BIN) run wf/fail.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)"
 
 path-input: compile
 	mkdir -p work
 	date > work/COPY_THIS.md
 	echo "$(TEST_URI)"
-	$(NF_BIN) run $(PATH_NF) -profile standard -plugins $(PROJECT) --outdir "./results"
+	$(NF_BIN) run wf/main.path.nf -profile standard -plugins $(PROJECT) --outdir "./results"
 
 tower-test: $(NF_BIN)
 	$(NF_BIN) run "https://github.com/quiltdata/nf-quilt" -name local_einstein  -with-tower -r main -latest --pub "$(TEST_URI)"

README-DEV.md

@@ -0,0 +1,139 @@
+# Developer Documentation
+
+## Using Pre-Release Versions
+
+Occasionally we will release beta versions of the plugin that are not yet available in the Nextflow plugin registry. You can help test these versions as follows:
+
+- Set the `NXF_PLUGINS_TEST_REPOSITORY` environment variable to the URL of the plugin's metadata file
+- Specify the plugin version in the `plugins` section of your `nextflow.config` file
+
+From the command-line, do, e.g.:
+
+```bash
+export LOG4J_DEBUG=true  # for verbose logging
+export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.8.5/nf-quilt-0.8.5-meta.json
+nextflow run main.nf -plugins [email protected]
+```
+
+For Tower, you set the environment variables in the "Pre-run script".
+
+![Example Tower Pre-run Script](./images/tower-beta.png)
+
+## Using Development Versions
+
+If you want to use edge versions of nf-quilt, you must run it with a development
+version of `nextflow`. The simplest way to do that is to pull them both directly
+from GitHub:
+
+```bash
+git clone https://github.com/nextflow.io/nextflow.git
+git clone https://github.com/quiltdata/nf-quilt.git
+cd ./nf-quilt
+```
+
+### Verifying Nextflow
+
+If this is your first time using Nextflow, you may also need to install a recent
+[version of Java](https://www.java.com/en/download/help/download_options.html)
+for your platform. Nextflow itself will take care of all the other dependencies.
+
+You can verify and compile Nextflow with:
+
+```bash
+make nextflow
+```
+
+## Running from Git
+
+To quickly run `nf-quilt` from this GitHub repository:
+
+```bash
+# install and compiles dependencies, then test
+make test-all 
+# create "test/hurdat" package on s3://$WRITE_BUCKET
+make pkg-test WRITE_BUCKET=your-writeablebucket 
+```
+
+This ensures you have properly installed Nextflow and configured your local
+<--markdownlint-disable-next-line MD041-->
+[AWS credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html).
+
+You can also set WRITE_BUCKET and other parameters in a `.env` file in the
+project root, and they will be automatically read by the Makefile.
+
+### Running a Pipeline Locally
+
+From inside the `nf-quilt` directory, call `./launch.sh` with a path to your
+pipeline.
+
+For example, with a standard `nf-core` pipeline like `sarek`:
+
+```bash
+./launch.sh run nf-core/sarek -profile test,docker -plugins nf-quilt --outdir "quilt+s3://bucket#package=nf-quilt/sarek"
+```
+
+Otherwise, replace `nf-core/sarek` with the local path to your pipeline's `.nf`
+file (be sure to rename the `outdir` parameter if you use different convention).
+For example:
+
+```bash
+   ./launch.sh run ./main.nf -profile standard -plugins $(PROJECT) --outdir "quilt+s3://bucket#package=test/hurdat"
+```
+
+### Unit Testing
+
+You can cleanly compile and run all unit tests with:
+
+```bash
+make check
+```
+
+To show the output of the tests, use:
+
+```bash
+make verify
+```
+
+To fast-fail on the first failing test, use:
+
+```bash
+make fast
+```
+
+## Publishing the Plugin for Others to Use
+
+If your system is properly configured, use `make publish` to package, upload,
+and publish the plugin.
+
+Otherwise, follow these steps:
+
+1. Create a file named `gradle.properties` in the project root containing the
+   following attributes (this file should not be committed to Git):
+
+   - `github_organization`: the GitHub organisation where the plugin repository
+     is hosted.
+   - `github_username`: The GitHub username granting access to the plugin
+     repository.
+   - `github_access_token`: The GitHub access token required to upload and
+     commit changes to the plugin repository.
+   - `github_commit_email`: The email address associated with your GitHub
+     account.
+
+2. Use the following command to package and create a release for your plugin on
+   GitHub:
+
+   ```bash
+   ./gradlew :plugins:nf-quilt:upload
+   ```
+
+3. Fork the [nextflow-io/plugins](https://github.com/nextflow-io/plugins)
+   repository to one you can write to
+
+4. Use the following command to publish your plugin to your fork:
+
+   ```bash
+   ./gradlew :plugins:publishIndex
+   ```
+
+5. Create a pull request to push your changes back to
+   [nextflow-io/plugins](https://github.com/nextflow-io/plugins/blob/main/plugins.json)