From 93b6a4faff60ebd116fe7ca7b16fbc66956be705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hampus=20N=C3=A4sstr=C3=B6m?= Date: Wed, 6 Mar 2024 09:29:22 +0100 Subject: [PATCH] Initial commit --- .github/workflows/docker-publish.yml | 101 ++++++++ .github/workflows/initialize.yml | 124 ++++++++++ .gitignore | 161 +++++++++++++ Dockerfile | 8 + LICENSE | 201 ++++++++++++++++ README.md | 44 ++++ nomad-oasis_files/nomad-oasis.zip | Bin 0 -> 4918 bytes .../nomad-oasis/configs/nginx.conf | 82 +++++++ .../nomad-oasis/configs/nomad.yaml | 34 +++ .../nomad-oasis/docker-compose.yaml | 217 ++++++++++++++++++ nomad.yaml | 34 +++ plugins.txt | 2 + template_README.md | 86 +++++++ 13 files changed, 1094 insertions(+) create mode 100644 .github/workflows/docker-publish.yml create mode 100644 .github/workflows/initialize.yml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 README.md create mode 100644 nomad-oasis_files/nomad-oasis.zip create mode 100644 nomad-oasis_files/nomad-oasis/configs/nginx.conf create mode 100644 nomad-oasis_files/nomad-oasis/configs/nomad.yaml create mode 100644 nomad-oasis_files/nomad-oasis/docker-compose.yaml create mode 100644 nomad.yaml create mode 100644 plugins.txt create mode 100644 template_README.md diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..a22ccf3 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,101 @@ +name: Docker + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +on: + schedule: + - cron: '00 03 * * 0' + push: + branches: [ "main" ] + # Publish semver tags as releases. + tags: [ 'v*.*.*' ] + pull_request: + branches: [ "main" ] + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + + +jobs: + + build: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Install the cosign tool except on PR + # https://github.com/sigstore/cosign-installer + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1 + with: + cosign-release: 'v2.1.1' + + # Set up BuildKit Docker container builder to be able to build + # multi-platform images and export cache + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} diff --git a/.github/workflows/initialize.yml b/.github/workflows/initialize.yml new file mode 100644 index 0000000..ac1506e --- /dev/null +++ b/.github/workflows/initialize.yml @@ -0,0 +1,124 @@ + +name: Template Repository Initialization + +on: + # Triggers the workflow on creation of repository + create: + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + +permissions: + contents: write + +jobs: + initialize_repository: + if: github.repository != 'FAIRmat-NFDI/nomad-distribution-template' + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + # Replaces the template repository name in the README with the new one + - name: Update README + run: | + sed -i "s|GITHUB_REPOSITORY_OWNER|${{ github.repository_owner }}|g" template_README.md + sed -i "s|GITHUB_REPOSITORY|${{ github.repository }}|g" template_README.md + mv template_README.md README.md + + # Replaces the template repository name in the docker config file with the new one + - name: Update docker-compose.yml + run: | + cd nomad-oasis_files + sed -i "s|FAIRMAT_NFDI/nomad-distribution-template|${{ github.repository }}|g" docker-compose.yml + zip -r nomad-oasis.zip nomad-oasis + cd .. + + # Deletes this workflow file to prevent it from running on branch creation + - name: Delete initialization workflow + run: rm .github/workflows/initialize.yml + + # Commits all changes + - name: Commit repository initialization + run: | + git config --global user.name github-actions + git config --global user.email github-actions@github.com + git commit -am "Repository initialization" + git push + + build: + if: github.repository != 'FAIRmat-NFDI/nomad-distribution-template' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Install the cosign tool except on PR + # https://github.com/sigstore/cosign-installer + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1 + with: + cosign-release: 'v2.1.1' + + # Set up BuildKit Docker container builder to be able to build + # multi-platform images and export cache + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f21df43 --- /dev/null +++ b/.gitignore @@ -0,0 +1,161 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.pyenv +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2ece209 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:develop +USER root +RUN apt-get update +RUN apt-get -y install git +USER nomad +COPY plugins.txt plugins.txt +RUN pip install -r plugins.txt +COPY nomad.yaml nomad.yaml \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..3647ce1 --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +![docker image](https://github.com/FAIRmat-NFDI/nomad-distribution-template/actions/workflows/docker-publish.yml/badge.svg) + +> [!IMPORTANT] +> The templated repository will run a GitHub action on creation which might take a few momements. +> After the workflow finishes you should refresh the page and this message should disappear. + +# NOMAD Oasis Distribution *Template* +This repository is a template for creating your own custom NOMAD Oasis distribution image. +Click [here](https://github.com/new?template_name=nomad-distribution-template&template_owner=FAIRmat-NFDI) +to use this template, or click the `Use this template` button in the upper right corner of +the main GitHub page for this template. + +## Deploying the image + +To deploy this NOMAD Oasis image you should follow the instructions on [nomad-lab.eu/prod/v1/docs/oasis/install.html](https://nomad-lab.eu/prod/v1/docs/oasis/install.html) but replace the Docker image in `docker-compose.yaml` with `ghcr.io/FAIRmat-NFDI/nomad-distribution-template:main` for the services `worker`, `app`, `north`, and `logtransfer`. + +Remember to also update the `nomad.yaml` config file to include the new plugins. + +### Quick-start + +- Find a linux computer. +- Make sure you have [docker](https://docs.docker.com/engine/install/) installed. +Docker nowadays comes with `docker compose` build in. Prior, you needed to +install the stand alone [docker-compose](https://docs.docker.com/compose/install/). +- Download the modified configuration files [nomad-oasis.zip](nomad-oasis_files/nomad-oasis.zip) from this repository. +- Run the following commands (skip `chown` on MacOS and Windows computers) + + +```sh +unzip nomad-oasis.zip +cd nomad-oasis +sudo chown -R 1000 .volumes +docker compose pull +docker compose up -d +curl localhost/nomad-oasis/alive +``` + +- Open [http://localhost/nomad-oasis](http://localhost/nomad-oasis) in your browser. + +To run NORTH (the NOMAD Remote Tools Hub), the `hub` container needs to run docker and +the container has to be run under the docker group. You need to replace the default group +id `991` in the `docker-compose.yaml`'s `hub` section with your systems docker group id. +Run `id` if you are a docker user, or `getent group | grep docker` to find our your +systems docker gid. The user id 1000 is used as the nomad user inside all containers. diff --git a/nomad-oasis_files/nomad-oasis.zip b/nomad-oasis_files/nomad-oasis.zip new file mode 100644 index 0000000000000000000000000000000000000000..1d6684ef9dc7ab7c3a6d51f626071c9693330c97 GIT binary patch literal 4918 zcmbW5cRbbmAIHxjazbV@jv2|`nb~#h?43Gt&an>}q145Rtcnm$%E%}qM@HyW#4#!> z93v|?dnP0EcT_jyru(aN9-r^yd(I!v*Zc8(KVI+idLMml;(Z4In=iB)-2Cg!_X`C8 z3~)udK<$K(P!9(W5fft&fLMxU1h#p+e8>U7eIvvGz}JiZ4+{S-3jHq1d88}c0r8!7 zj(G$IC7hqo=m}Mj0ZbI&=8><}HJXV406{VUfNc+&E5gCmM|hL{waS2lBd}Y9vx}Q+ z;^NdTO1*yXET^1Wmq88434%8iKNdlGoRKBSpG@NL_(WC|kFFtYrqfC{Z>KLNNsu*t znf7DpFk8>4-g7WL=xF;RpA2zIdd*uJoT3<=Fv^8YwfCvg~ObiaCDwE_Me&)tZ(t8*x6mw zsg|V^gG6;|@|_r2H2aY5rJ!xp3iqF3DvfW?fmI3<4Mu-{jx7^_k>bGkJV+^YX(8)W ze2`7_WVEf5(`u;yQ3^gFCMU)FVUuTjF9SIXPX@{RO0Z69LV@nHT3GetjH zXDU59EoGR|BXK>~Gmkzz8-aM9J%3g9PHS(~(A@~B%FkbD6r!Wdz$a_gKVtU1y4Gsc zn%0|J+SnY}r{QiWLaJYO<*)k{9W&t%1m}Ce`uU8AQfyZZHoz`$Ad=oxTdzzt7^fAP zSv5(El;`>Krq1CmF~Vl7P%AUwZ+%oABYtgafjSQNg+I7MG>Rk2!BPO4@}39HV9)8q z)o5>}FPKP`n(%=(OZG-^h5xkI1L(k2oP>((!5})r@^X{}{vqZy7n}B&B7EXt9eOT@ zH5k_?A?ZC|Wc#7o)+@lElv=SrDP%@9M5@a~!-comNZPG;dN|~upowr_ETq`J$bCj) zHTdPL(`Pl4#qQ)CKC?1fJf$>L18k!*6j3&Ma0L<=HXvVq{asPrv?B(~MGj_3&}oj= zswvd7*GW`Cx^V=uj{~jQYxXZmqh+y3IJ?e#`K1uo&!icvJ*5*TSU>e2T{*jHyKhF} zCfH7uHSd8Jkl-7FEB7?YrlEy>p)Sr}jbcnN-=+z786`6MwQIIAMew2mNqG))G&ODQ zBc!;v1ZhSR8mDH``#e~)8_d@-N(7x~T%Wxrdc{7}Q=pzUkl}Wwr@%#=wrDoCeB}}c zJ+P&!qTZK$teV)N>rnNA&74%I^Bum1+8YVCQ0jNM7%SKf%-uv&VR%;lyEm<421;ZF zL4%TusyH6**&F2HUif16WjZz|`^I4gJzXJ@OuZtlqG70gc?S02JW#~z-C6W2X|#8@bW6&I2k~C%pu{Smrr75 zR#Ii-)y`?^;hYB~_wKgMs*m+nT zI`P@yrCZa_8jufpL5=lXP2duimiN6Y$7iy)jAv+rhhjMISD|uM>Svr8LjO)vf5|dHBmqrR zhy)Fj9+rHqk=&lTPvkn*Kt``zi{+6@j2w#(%Yk@=>P%dVaj=N@G|^Z`!))&aR5xyzciV)w6juxg^XGC{~S$ z>=LqZW#>T+rSjXV=aWU8;uR7y2$&bIbk}kEvCZM zL61b~xt`><(63B5WW#(F8BC!k=W1F8ePh{VjcK?~M+P#H(IpCxRJ=dd)c7Xu%PN}F ztYZ!M!aL+n9^NbC27dBpqX~x~%+Slf-)0O}P|%`q<6=l@{&)jh2Fx)lokn6pWKO61 z9=8nT=M*A4MzJ$?<|y>dfY+Wwy=X4SxpLZ-(Qwn1Ydmcw9ftH_Wf6e=j;U$q-~|ek z^bnbKuJEv)^%acf9f(rALP^odY6kyQ9q**U2!j!I@eQ_Dj+`grPlY!j;Oe>LnM!*3 zqQQrIo~JK_aMf4bvk*?6WD4ZAh#Q zjKggy9b~^w#@dZwV*%^XV=~q`$d(jq64%oi4S&ay0#^3D;=Au;L=+t*+0RzEA#PkW zmDfX~uBuDsCo^*(I#23>=_*xA7Cnl1#wuY+gNJqGO&W;^mt&qgZ8t+hLH$Y|MbaFR z2~F#qq*eH>j9QJ)&w?m2$GVNjs z&%_nX0u^e65W&}usY;&VRf(eql0b724zsdE=89L z1-n)GS52l$=ebh?IaD0p7{oU=&5nZ)P(oi~c(2Ek8};PhRZJwZV**(S@~d3Vzkq?R zFCL1Bv^kMk<~saHdLbD5Q1F12oO*3p884mulp9&wtMQwUzW5yG#iNCc)|{p)oDuU4 z1*U%$3-moU!Cz4w(uyxDRPD7~ZpRHMUAm+1P}|ve3gV(6vNDn5U-*RiX}5LJ1ZkVXLHQRxxBjjaf;n8PyB7~G08N9u_r!n;GVy>9ME>W z);u{4pvIy{lhn8&)AeyN5uYB%*XJBg_QJ<38xD|Olx;Yl61S*7qTlp8*S}nuZH(8aj!YT9azWa;!Y0O0D zti^x$XyXR5KhEsurw<=rGH&yxWZnR5#pd1N`_QhDFYJwU_Hu#!zbSk-j&Uy>d>irj zuX6ov;_iU)qd)@D6?wtaersR&EiU&jD*eXw@_=3V)2es>l5Vfs&CAx=;rt(#&GCz} zJv^a=^#t*UWbiM@o-S^GNM+fJ>VkAdApZjjBw{1pYiDHqzCBU{CRH|XcW#g*-!{ly z0YzVXi}>&TaXazUF5=cUNg)3u$bKX5wM%{@Zzkpk0Wo`!w{MmNjm^#Sr`7Kj!?$Z- z3Bi2(^Z2Lwuit%-aRiOcQ2q;zf2YIk8exRYxLxC?S&<;}lbpD{4ZH1r2(Y~k{^UDF ze%Q#wn1KH2{{vec-i>D5fqt_W`s<|neLC9?Ivd3vg>Rc$zpuf)L&N(&HU61kzpuo; zL#g!_N`;l!;w%=C)?@$T(h03=n_xlPgI~02MQut@$?Il1$N*E0Q QO2T_;9{?bJaP!mu07%*oW&i*H literal 0 HcmV?d00001 diff --git a/nomad-oasis_files/nomad-oasis/configs/nginx.conf b/nomad-oasis_files/nomad-oasis/configs/nginx.conf new file mode 100644 index 0000000..1b8531f --- /dev/null +++ b/nomad-oasis_files/nomad-oasis/configs/nginx.conf @@ -0,0 +1,82 @@ +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +server { + listen 80; + server_name localhost; + proxy_set_header Host $host; + + gzip_min_length 1000; + gzip_buffers 4 8k; + gzip_http_version 1.0; + gzip_disable "msie6"; + gzip_vary on; + gzip on; + gzip_proxied any; + gzip_types + text/css + text/javascript + text/xml + text/plain + application/javascript + application/x-javascript + application/json; + + location / { + proxy_pass http://app:8000; + } + + location ~ /nomad-oasis\/?(gui)?$ { + rewrite ^ /nomad-oasis/gui/ permanent; + } + + location /nomad-oasis/gui/ { + proxy_intercept_errors on; + error_page 404 = @redirect_to_index; + proxy_pass http://app:8000; + } + + location @redirect_to_index { + rewrite ^ /nomad-oasis/gui/index.html break; + proxy_pass http://app:8000; + } + + location ~ \/gui\/(service-worker\.js|meta\.json)$ { + add_header Last-Modified $date_gmt; + add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0'; + if_modified_since off; + expires off; + etag off; + proxy_pass http://app:8000; + } + + location ~ /api/v1/uploads(/?$|.*/raw|.*/bundle?$) { + client_max_body_size 35g; + proxy_request_buffering off; + proxy_pass http://app:8000; + } + + location ~ /api/v1/.*/download { + proxy_buffering off; + proxy_pass http://app:8000; + } + + location /nomad-oasis/north/ { + client_max_body_size 500m; + proxy_pass http://north:9000; + + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # websocket headers + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header X-Scheme $scheme; + + proxy_buffering off; + } +} diff --git a/nomad-oasis_files/nomad-oasis/configs/nomad.yaml b/nomad-oasis_files/nomad-oasis/configs/nomad.yaml new file mode 100644 index 0000000..6b03954 --- /dev/null +++ b/nomad-oasis_files/nomad-oasis/configs/nomad.yaml @@ -0,0 +1,34 @@ +services: + api_host: 'localhost' + api_base_path: '/nomad-oasis' + +oasis: + is_oasis: true + uses_central_user_management: true + +north: + jupyterhub_crypt_key: '978bfb2e13a8448a253c629d8dd84ff89587f30e635b753153960930cad9d36d' + +meta: + deployment: 'oasis' + deployment_url: 'https://my-oasis.org/api' + maintainer_email: 'me@my-oasis.org' + +logstash: + enable: false + +mongo: + db_name: nomad_oasis_v1 + +elastic: + entries_index: nomad_oasis_entries_v1 + materials_index: nomad_oasis_materials_v1 + +plugins: + options: + schemas/nomad_material_processing: + python_package: nomad_material_processing + schemas/nomad_measurements: + python_package: nomad_measurements + parsers/nomad_measurements/xrd: + python_package: nomad_measurements.xrd.parser \ No newline at end of file diff --git a/nomad-oasis_files/nomad-oasis/docker-compose.yaml b/nomad-oasis_files/nomad-oasis/docker-compose.yaml new file mode 100644 index 0000000..13f6179 --- /dev/null +++ b/nomad-oasis_files/nomad-oasis/docker-compose.yaml @@ -0,0 +1,217 @@ +version: "3" + +services: + # broker for celery + rabbitmq: + restart: unless-stopped + image: rabbitmq:3.11.5 + container_name: nomad_oasis_rabbitmq + environment: + - RABBITMQ_ERLANG_COOKIE=SWQOKODSQALRPCLNMEQG + - RABBITMQ_DEFAULT_USER=rabbitmq + - RABBITMQ_DEFAULT_PASS=rabbitmq + - RABBITMQ_DEFAULT_VHOST=/ + volumes: + - rabbitmq:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "--silent", "--quiet", "ping"] + interval: 10s + timeout: 10s + retries: 30 + start_period: 10s + + # the search engine + elastic: + restart: unless-stopped + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1 + container_name: nomad_oasis_elastic + environment: + - ES_JAVA_OPTS=-Xms512m -Xmx512m + - discovery.type=single-node + volumes: + - elastic:/usr/share/elasticsearch/data + healthcheck: + test: + - "CMD" + - "curl" + - "--fail" + - "--silent" + - "http://elastic:9200/_cat/health" + interval: 10s + timeout: 10s + retries: 30 + start_period: 60s + + # the user data db + mongo: + restart: unless-stopped + image: mongo:5.0.6 + container_name: nomad_oasis_mongo + environment: + - MONGO_DATA_DIR=/data/db + - MONGO_LOG_DIR=/dev/null + volumes: + - mongo:/data/db + - ./.volumes/mongo:/backup + command: mongod --logpath=/dev/null # --quiet + healthcheck: + test: + - "CMD" + - "mongo" + - "mongo:27017/test" + - "--quiet" + - "--eval" + - "'db.runCommand({ping:1}).ok'" + interval: 10s + timeout: 10s + retries: 30 + start_period: 10s + + # nomad worker (processing) + worker: + restart: unless-stopped + image: ghcr.io/FAIRmat-NFDI/nomad-distribution-template:main + container_name: nomad_oasis_worker + environment: + NOMAD_SERVICE: nomad_oasis_worker + NOMAD_RABBITMQ_HOST: rabbitmq + NOMAD_ELASTIC_HOST: elastic + NOMAD_MONGO_HOST: mongo + NOMAD_LOGSTASH_HOST: logtransfer + depends_on: + rabbitmq: + condition: service_healthy + elastic: + condition: service_healthy + mongo: + condition: service_healthy + volumes: + # - ./configs/nomad.yaml:/app/nomad.yaml + - ./.volumes/fs:/app/.volumes/fs + command: python -m celery -A nomad.processing worker -l info -Q celery + + # nomad app (api + proxy) + app: + restart: unless-stopped + image: ghcr.io/FAIRmat-NFDI/nomad-distribution-template:main + container_name: nomad_oasis_app + environment: + NOMAD_SERVICE: nomad_oasis_app + NOMAD_SERVICES_API_PORT: 80 + NOMAD_FS_EXTERNAL_WORKING_DIRECTORY: "$PWD" + NOMAD_RABBITMQ_HOST: rabbitmq + NOMAD_ELASTIC_HOST: elastic + NOMAD_MONGO_HOST: mongo + NOMAD_LOGSTASH_HOST: logtransfer + NOMAD_NORTH_HUB_HOST: north + depends_on: + rabbitmq: + condition: service_healthy + elastic: + condition: service_healthy + mongo: + condition: service_healthy + north: + condition: service_started + volumes: + # - ./configs/nomad.yaml:/app/nomad.yaml + - ./.volumes/fs:/app/.volumes/fs + command: ./run.sh + healthcheck: + test: + - "CMD" + - "curl" + - "--fail" + - "--silent" + - "http://localhost:8000/-/health" + interval: 10s + timeout: 10s + retries: 30 + start_period: 10s + + # nomad remote tools hub (JupyterHUB, e.g. for AI Toolkit) + north: + restart: unless-stopped + image: ghcr.io/FAIRmat-NFDI/nomad-distribution-template:main + container_name: nomad_oasis_north + environment: + NOMAD_SERVICE: nomad_oasis_north + NOMAD_NORTH_DOCKER_NETWORK: nomad_oasis_network + NOMAD_NORTH_HUB_CONNECT_IP: north + NOMAD_NORTH_HUB_IP: "0.0.0.0" + NOMAD_NORTH_HUB_HOST: north + NOMAD_SERVICES_API_HOST: app + NOMAD_FS_EXTERNAL_WORKING_DIRECTORY: "$PWD" + NOMAD_RABBITMQ_HOST: rabbitmq + NOMAD_ELASTIC_HOST: elastic + NOMAD_MONGO_HOST: mongo + volumes: + # - ./configs/nomad.yaml:/app/nomad.yaml + - ./.volumes/fs:/app/.volumes/fs + - /var/run/docker.sock:/var/run/docker.sock + user: '1000:991' + command: python -m nomad.cli admin run hub + healthcheck: + test: + - "CMD" + - "curl" + - "--fail" + - "--silent" + - "http://localhost:8081/nomad-oasis/north/hub/health" + interval: 10s + timeout: 10s + retries: 30 + start_period: 10s + + # nomad logtransfer + # to enable the logtransfer service run "docker compose --profile with_logtransfer up" + logtransfer: + restart: unless-stopped + image: ghcr.io/FAIRmat-NFDI/nomad-distribution-template:main + container_name: nomad_oasis_logtransfer + environment: + NOMAD_SERVICE: nomad_oasis_logtransfer + NOMAD_ELASTIC_HOST: elastic + NOMAD_MONGO_HOST: mongo + depends_on: + elastic: + condition: service_healthy + mongo: + condition: service_healthy + volumes: + # - ./configs/nomad.yaml:/app/nomad.yaml + - ./.volumes/fs:/app/.volumes/fs + command: python -m nomad.cli admin run logtransfer + profiles: ["with_logtransfer"] + + # nomad proxy (a reverse proxy for nomad) + proxy: + restart: unless-stopped + image: nginx:1.13.9-alpine + container_name: nomad_oasis_proxy + command: nginx -g 'daemon off;' + volumes: + - ./configs/nginx.conf:/etc/nginx/conf.d/default.conf + depends_on: + app: + condition: service_healthy + worker: + condition: service_started # TODO: service_healthy + north: + condition: service_healthy + ports: + - 80:80 + +volumes: + mongo: + name: "nomad_oasis_mongo" + elastic: + name: "nomad_oasis_elastic" + rabbitmq: + name: "nomad_oasis_rabbitmq" + keycloak: + name: "nomad_oasis_keycloak" + +networks: + default: + name: nomad_oasis_network diff --git a/nomad.yaml b/nomad.yaml new file mode 100644 index 0000000..6b03954 --- /dev/null +++ b/nomad.yaml @@ -0,0 +1,34 @@ +services: + api_host: 'localhost' + api_base_path: '/nomad-oasis' + +oasis: + is_oasis: true + uses_central_user_management: true + +north: + jupyterhub_crypt_key: '978bfb2e13a8448a253c629d8dd84ff89587f30e635b753153960930cad9d36d' + +meta: + deployment: 'oasis' + deployment_url: 'https://my-oasis.org/api' + maintainer_email: 'me@my-oasis.org' + +logstash: + enable: false + +mongo: + db_name: nomad_oasis_v1 + +elastic: + entries_index: nomad_oasis_entries_v1 + materials_index: nomad_oasis_materials_v1 + +plugins: + options: + schemas/nomad_material_processing: + python_package: nomad_material_processing + schemas/nomad_measurements: + python_package: nomad_measurements + parsers/nomad_measurements/xrd: + python_package: nomad_measurements.xrd.parser \ No newline at end of file diff --git a/plugins.txt b/plugins.txt new file mode 100644 index 0000000..041a1f0 --- /dev/null +++ b/plugins.txt @@ -0,0 +1,2 @@ +nomad-measurements +nomad-material-processing diff --git a/template_README.md b/template_README.md new file mode 100644 index 0000000..93282b5 --- /dev/null +++ b/template_README.md @@ -0,0 +1,86 @@ +![docker image](https://github.com/GITHUB_REPOSITORY/actions/workflows/docker-publish.yml/badge.svg) + +# GITHUB_REPOSITORY_OWNER's NOMAD Oasis Distribution + +This is the NOMAD Oasis distribution of GITHUB_REPOSITORY_OWNER. +Below are instructions for how to [deploy this image](#deploying-the-image) and how to +customize it through [adding plugins](#adding-a-plugin). + +> [!IMPORTANT] +> Depending on the settings of the owner of this repository, the distributed image might +> be private and require authentication to pull. +> If you are the owner you should make sure that your organization settings allow public +> packages and after that set this package public. +> You can read more about this in the GitHub docs [here](https://docs.github.com/en/packages/learn-github-packages/configuring-a-packages-access-control-and-visibility). + +## Deploying the image + +To deploy this NOMAD Oasis image you should follow the instructions on +[nomad-lab.eu/prod/v1/docs/oasis/install.html](https://nomad-lab.eu/prod/v1/docs/oasis/install.html) +but replace the Docker image in `docker-compose.yaml` with `ghcr.io/GITHUB_REPOSITORY:main` +for the services `worker`, `app`, `north`, and `logtransfer`. + +Remember to also update the `nomad.yaml` config file to include the new plugins. + +### Quick-start + +- Find a linux computer. +- Make sure you have [docker](https://docs.docker.com/engine/install/) installed. +Docker nowadays comes with `docker compose` build in. Prior, you needed to +install the stand alone [docker-compose](https://docs.docker.com/compose/install/). +- Download the modified configuration files [nomad-oasis.zip](nomad-oasis_files/nomad-oasis.zip) from this repository. +- Run the following commands (skip `chown` on MacOS and Windows computers) + + +```sh +unzip nomad-oasis.zip +cd nomad-oasis +sudo chown -R 1000 .volumes +docker compose pull +docker compose up -d +curl localhost/nomad-oasis/alive +``` + +- Open [http://localhost/nomad-oasis](http://localhost/nomad-oasis) in your browser. + +To run NORTH (the NOMAD Remote Tools Hub), the `hub` container needs to run docker and +the container has to be run under the docker group. You need to replace the default group +id `991` in the `docker-compose.yaml`'s `hub` section with your systems docker group id. +Run `id` if you are a docker user, or `getent group | grep docker` to find our your +systems docker gid. The user id 1000 is used as the nomad user inside all containers. + +## Adding a plugin + +To add a new plugin to the docker image you should add it to the [plugins.txt](plugins.txt) file. + +Here you can put either plugins distributed to PyPI, e.g. +``` +nomad-material-processing +``` +or plugins in a git repository with either the commit hash +``` +git+https://github.com/FAIRmat-NFDI/nomad-measurements.git@71b7e8c9bb376ce9e8610aba9a20be0b5bce6775 +``` +or with a tag +``` +git+https://github.com/FAIRmat-NFDI/nomad-measurements.git@v0.0.4 +``` +To add a plugin in a subdirectory of a git repository you can use the `subdirectory` option, e.g. +``` +git+https://github.com/FAIRmat-NFDI/AreaA-data_modeling_and_schemas.git@30fc90843428d1b36a1d222874803abae8b1cb42#subdirectory=PVD/PLD/jeremy_ikz/ikz_pld_plugin +``` + +If the plugin is new, you also need to add it under `plugins` in the [nomad.yaml](nomad.yaml) +config file that will be included in the image. +For example, if you have added a schema plugin `nomad_material_processing` you should add +the following: + +```yaml +plugins: + options: + schemas/nomad_material_processing: + python_package: nomad_material_processing +``` + +Once the changes have been committed to the main branch, the new image will automatically +be generated. \ No newline at end of file