From d18f6649da736eb51646b99c5725ce5ae1997cb5 Mon Sep 17 00:00:00 2001 From: joss Date: Thu, 28 Nov 2024 00:07:26 +0000 Subject: [PATCH] feat: add chart with an example and ci --- .github/CODEOWNERS | 2 + .github/commitlint.config.mjs | 16 ++ .github/pull_request_template.md | 14 ++ .github/renovate.js | 78 +++++++++ .github/workflows/kargo-pipelines-chart.yaml | 51 ++++++ .github/workflows/pr-title-commitlint.yaml | 19 ++ .github/workflows/renovate.yaml | 23 +++ .gitignore | 164 +----------------- charts/kargo-pipelines/.helmignore | 24 +++ charts/kargo-pipelines/Chart.yaml | 6 + .../kargo-pipelines/templates/argo/app.yaml | 59 +++++++ .../templates/argo/namespace.yaml | 34 ++++ .../templates/argo/project.yaml | 100 +++++++++++ .../templates/extra-manifests.yaml | 6 + .../templates/kargo/project.yaml | 21 +++ .../templates/kargo/stage.yaml | 107 ++++++++++++ .../templates/kargo/warehouse.yaml | 38 ++++ charts/kargo-pipelines/values.yaml | 76 ++++++++ example/kargo-demo/base/kustomization.yaml | 5 + .../kargo-demo/base/manifests/airflow.yaml | 67 +++++++ .../deployments/dev-0/kustomization.yaml | 14 ++ .../dev-0/manifests/extra-configmap.yaml | 7 + .../dev-0/patches/applications.yaml | 7 + .../deployments/dev-1/kustomization.yaml | 13 ++ .../dev-1/patches/applications.yaml | 7 + .../deployments/uat/kustomization.yaml | 13 ++ .../deployments/uat/patches/applications.yaml | 7 + example/kargo-demo/kargo-pipeline.yaml | 84 +++++++++ 28 files changed, 900 insertions(+), 162 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/commitlint.config.mjs create mode 100644 .github/pull_request_template.md create mode 100644 .github/renovate.js create mode 100644 .github/workflows/kargo-pipelines-chart.yaml create mode 100644 .github/workflows/pr-title-commitlint.yaml create mode 100644 .github/workflows/renovate.yaml create mode 100644 charts/kargo-pipelines/.helmignore create mode 100644 charts/kargo-pipelines/Chart.yaml create mode 100644 charts/kargo-pipelines/templates/argo/app.yaml create mode 100644 charts/kargo-pipelines/templates/argo/namespace.yaml create mode 100644 charts/kargo-pipelines/templates/argo/project.yaml create mode 100644 charts/kargo-pipelines/templates/extra-manifests.yaml create mode 100644 charts/kargo-pipelines/templates/kargo/project.yaml create mode 100644 charts/kargo-pipelines/templates/kargo/stage.yaml create mode 100644 charts/kargo-pipelines/templates/kargo/warehouse.yaml create mode 100644 charts/kargo-pipelines/values.yaml create mode 100644 example/kargo-demo/base/kustomization.yaml create mode 100644 example/kargo-demo/base/manifests/airflow.yaml create mode 100644 example/kargo-demo/deployments/dev-0/kustomization.yaml create mode 100644 example/kargo-demo/deployments/dev-0/manifests/extra-configmap.yaml create mode 100644 example/kargo-demo/deployments/dev-0/patches/applications.yaml create mode 100644 example/kargo-demo/deployments/dev-1/kustomization.yaml create mode 100644 example/kargo-demo/deployments/dev-1/patches/applications.yaml create mode 100644 example/kargo-demo/deployments/uat/kustomization.yaml create mode 100644 example/kargo-demo/deployments/uat/patches/applications.yaml create mode 100644 example/kargo-demo/kargo-pipeline.yaml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..30bab28 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +* @JossWhittle +.github/ @SwanseaUniversityMedical/devops-maintainers diff --git a/.github/commitlint.config.mjs b/.github/commitlint.config.mjs new file mode 100644 index 0000000..af302bb --- /dev/null +++ b/.github/commitlint.config.mjs @@ -0,0 +1,16 @@ +import { RuleConfigSeverity } from '@commitlint/types'; + +export default { + extends: ['@commitlint/config-conventional'], + parserPreset: 'conventional-changelog-conventionalcommits', + rules: { + 'scope-enum': [RuleConfigSeverity.Error, 'always', [ + '', + 'ci', + 'deps', + 'kargo-pipelines', + 'kargo-pipelines-chart', + ]], + 'subject-case': [RuleConfigSeverity.Error, 'never', []], + } +}; diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..ef876fa --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,14 @@ +## :construction: Suggest a change + +A clear and concise description of what you are changing. + +## :memo: Pre-merge checklist + +Ready to merge? Do not merge until all checks are satisfied. +- [ ] :chart: Have all `required` CI checks passed on the most recent commit? +- [ ] :black_nib: Is the PR title a valid and meaningful conventional-commit message? ie. `type(scope): summary` +- [ ] :boom: Are `breaking changes` declared in the PR title in conventional-commit style? ie. `type!(scope): summary` +- [ ] :art: Does new code follow the code style of this project? +- [ ] :mag: Has new code been spellchecked and linted? +- [ ] :book: Have docs been updated where necessary? +- [ ] :poop: Have commits been checked for accidental file inclusions? diff --git a/.github/renovate.js b/.github/renovate.js new file mode 100644 index 0000000..8df248f --- /dev/null +++ b/.github/renovate.js @@ -0,0 +1,78 @@ +module.exports = { + + // Uncomment dryRun to test exotic config options without spamming dozens of + // pull requests onto a repo that you would then need to clean up... + //dryRun: "full", + + // Inherit default config options + extends: ["config:base"], + configMigration: true, + + // Force use of Conventional Commit messages to avoid Renovate not detecting them + semanticCommits: "enabled", + + // Disable limits on the number of pull requests that can be managed simultaneously + // since this can sometimes prevent security patches being suggested! + prHourlyLimit: 0, + prConcurrentLimit: 0, + + // Tell Renovate to re-create or rebase old pull requests when new commits have + // since been merged into main... + rebaseWhen: "behind-base-branch", + + // Set the default schedule for when pull requests will be created or updated. + // If Renovate is run outside of this schedule then it will skip updating pull + // requests for dependencies unless they override the schedule. + updateNotScheduled: false, + timezone: "Europe/London", + schedule: [ + "after 10pm", + "before 5am" + ], + + // This setting helps handle breaking changes to Renovate bot when its version changes. + ignorePrAuthor: true, + + // Automatically assign reviewers to pull requests based on who "owns" the source files + // that need to be updated as listed in the CODEOWNERS file in the project repo. + reviewersFromCodeOwners: true, + + // Auto discovery is dangerous, never blindly trust the scope of the token! + autodiscover: false, + // Instead, explicitly list the repos that we should manage pull requests on. + // This should realistically only be one repo, the project repo you are currently in. + // The default token "should" only have access to this repo... + repositories: [ + "SwanseaUniversityMedical/kargo-pipelines", + ], + + packageRules: [ + { + // Group all otherwise ungrouped minor and patch updates into a single PR + groupName: "all non-major dependencies", + groupSlug: "all-minor-patch", + matchPackageNames: ["*"], + matchUpdateTypes: ["minor", "patch"] + }, + { + // By default all major updates won't create PRs unless checked in the dep dashboard + matchUpdateTypes: ["major"], + dependencyDashboardApproval: true + }, + { + // Group all minor and patch workflows updates into a single PR + groupName: "workflows non-major dependencies", + groupSlug: "workflows-minor-patch", + matchPackageNames: ["SwanseaUniversityMedical/workflows"], + matchUpdateTypes: ["minor", "patch"], + schedule: ["at any time"] + }, + { + // Force major workflows updates to create PRs without dep dashboard being checked + matchPackageNames: ["SwanseaUniversityMedical/workflows"], + matchUpdateTypes: ["major"], + dependencyDashboardApproval: false, + schedule: ["at any time"] + } + ], +}; diff --git a/.github/workflows/kargo-pipelines-chart.yaml b/.github/workflows/kargo-pipelines-chart.yaml new file mode 100644 index 0000000..82e404c --- /dev/null +++ b/.github/workflows/kargo-pipelines-chart.yaml @@ -0,0 +1,51 @@ +name: kargo-pipelines chart + +on: + pull_request: + # Only consider PRs that change files for this asset, including ci scripts + paths: + - '.github/workflows/kargo-pipelines-chart.yaml' + - 'charts/kargo-pipelines/**' + push: + # Only release off of release and maintenance branches for this asset + branches: + - 'main' + # Only consider pushes that change files for this asset, including ci scripts + paths: + - '.github/workflows/kargo-pipelines-chart.yaml' + - 'charts/kargo-pipelines/**' + +permissions: + contents: write + pull-requests: write + actions: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + # Cancel early on pull requests if new commits are added, + # Don't cancel on release pushes + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + # Job name must be unique across repo to target + # branch protection rules "required checks" properly! + kargo-pipelines-chart: + uses: SwanseaUniversityMedical/workflows/.github/workflows/pr-and-release-chart.yaml@v2.0.4-charts + with: + job-name: kargo-pipelines-chart + comment-pr: "true" + comment-release: "true" + registry: ${{ vars.HARBOR_REGISTRY }} + registry-user: ${{ vars.HARBOR_USER }} + registry-project: ${{ vars.HARBOR_PROJECT }} + registry-repo: kargo-pipelines + release-tag-format: 'v${version}-kargo-pipelines-chart' + cosign-public-key: ${{ vars.COSIGN_PUBLIC_KEY }} + chart: charts/kargo-pipelines + test-command: | + helm template $CHART --debug + + secrets: + cosign-private-key: ${{ secrets.COSIGN_PRIVATE_KEY }} + cosign-password: ${{ secrets.COSIGN_PASSWORD }} + registry-token: ${{ secrets.HARBOR_TOKEN }} diff --git a/.github/workflows/pr-title-commitlint.yaml b/.github/workflows/pr-title-commitlint.yaml new file mode 100644 index 0000000..00f3009 --- /dev/null +++ b/.github/workflows/pr-title-commitlint.yaml @@ -0,0 +1,19 @@ +name: Commit Lint PR Title + +on: + pull_request: + # Run on all PRs whenever the title could have changed + types: + - opened + - reopened + - edited + - synchronize + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + # This linting can be cancelled if there is a newer commit to lint + cancel-in-progress: true + +jobs: + pr-title-commitlint: + uses: SwanseaUniversityMedical/workflows/.github/workflows/pr-title-commitlint.yaml@v1.4.2-pr-title-commitlint diff --git a/.github/workflows/renovate.yaml b/.github/workflows/renovate.yaml new file mode 100644 index 0000000..406d1e6 --- /dev/null +++ b/.github/workflows/renovate.yaml @@ -0,0 +1,23 @@ +name: Renovate +on: + workflow_dispatch: + repository_dispatch: + types: [renovate] + schedule: + - cron: '0 * * * *' + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + renovate: + runs-on: + labels: [ self-hosted, linux, x64 ] + group: heavy + + steps: + - uses: SwanseaUniversityMedical/workflows/.github/actions/renovate@v1.0.3-renovate + with: + token: ${{ secrets.RENOVATE_TOKEN }} + config: '.github/renovate.js' \ No newline at end of file diff --git a/.gitignore b/.gitignore index 82f9275..ef1df10 100644 --- a/.gitignore +++ b/.gitignore @@ -1,162 +1,2 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/latest/usage/project/#working-with-version-control -.pdm.toml -.pdm-python -.pdm-build/ - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea +local/** \ No newline at end of file diff --git a/charts/kargo-pipelines/.helmignore b/charts/kargo-pipelines/.helmignore new file mode 100644 index 0000000..0f4f9fb --- /dev/null +++ b/charts/kargo-pipelines/.helmignore @@ -0,0 +1,24 @@ + +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line.. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/kargo-pipelines/Chart.yaml b/charts/kargo-pipelines/Chart.yaml new file mode 100644 index 0000000..794c3ec --- /dev/null +++ b/charts/kargo-pipelines/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +appVersion: 1.0.0 +description: Reusable kargo pipelines. +name: kargo-pipelines +type: application +version: 0.0.0 diff --git a/charts/kargo-pipelines/templates/argo/app.yaml b/charts/kargo-pipelines/templates/argo/app.yaml new file mode 100644 index 0000000..8490ab5 --- /dev/null +++ b/charts/kargo-pipelines/templates/argo/app.yaml @@ -0,0 +1,59 @@ +# Generate an argo app for each stage +{{ range $stage := .Values.stages }} +# Only output an app for this stage if the stage is enabled +{{- if or (not (hasKey $stage "enabled")) $stage.enabled }} +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + # App name is suffixed by the stage name + name: {{ $.Release.Name }}-{{ $stage.name }} + # Place the stage apps in the root namespace for the project + namespace: {{ $.Release.Namespace }} + + finalizers: + - resources-finalizer.argocd.argoproj.io + + annotations: + # Give the stage within the kargo project permission to call argocd-update + # on the app that is generated by this appset for the given stage + kargo.akuity.io/authorized-stage: {{ $.Release.Name }}:{{ $stage.name }} + kargo-pipelines.ukserp.ac.uk/component: argo-app + {{- with $.Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $.Values.argocd.app.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + + labels: + component: argo-app + app: {{ $.Chart.Name }} + release: {{ $.Release.Name }} + chart: {{ printf "%s-%s" $.Chart.Name $.Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} + heritage: {{ $.Release.Service }} + {{- with $.Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $.Values.argocd.app.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + # Place the stage apps into the argocd project for the root app + project: {{ $.Release.Name }} + source: + # Deploy every manifest found on the deployment branch for this stage + # where the branch will be otherwise empty compared to main + repoURL: {{ $.Values.warehouse.valuesFreight.repoURL }} + targetRevision: deployment/{{ $.Release.Name }}-{{ $stage.name }} + path: . + destination: + server: https://kubernetes.default.svc + # Deploy everything for this stage into its own isolated namespace + namespace: {{ $.Release.Namespace }}-{{ $stage.name }} + syncPolicy: + automated: + prune: true + selfHeal: true +{{- end }} +{{ end }} \ No newline at end of file diff --git a/charts/kargo-pipelines/templates/argo/namespace.yaml b/charts/kargo-pipelines/templates/argo/namespace.yaml new file mode 100644 index 0000000..c4acbf1 --- /dev/null +++ b/charts/kargo-pipelines/templates/argo/namespace.yaml @@ -0,0 +1,34 @@ +# Generate an isolated namespace for each stage +# We make namespaces explicitly so that they can be deleted wholesale +{{ range $stage := .Values.stages }} +# Only output a namespace for this stage if the stage is enabled +{{- if or (not (hasKey $stage "enabled")) $stage.enabled }} +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ $.Release.Namespace }}-{{ $stage.name }} + + annotations: + kargo-pipelines.ukserp.ac.uk/component: namespace + {{- with $.Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $.Values.namespace.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + + labels: + component: namespace + app: {{ $.Chart.Name }} + release: {{ $.Release.Name }} + chart: {{ printf "%s-%s" $.Chart.Name $.Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} + heritage: {{ $.Release.Service }} + {{- with $.Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $.Values.namespace.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} +{{ end }} \ No newline at end of file diff --git a/charts/kargo-pipelines/templates/argo/project.yaml b/charts/kargo-pipelines/templates/argo/project.yaml new file mode 100644 index 0000000..a8a6ad4 --- /dev/null +++ b/charts/kargo-pipelines/templates/argo/project.yaml @@ -0,0 +1,100 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: {{ .Release.Name }} + namespace: {{ .Values.argocd.namespace }} + + annotations: + kargo-pipelines.ukserp.ac.uk/component: argo-project + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.argocd.project.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + + labels: + argocd.argoproj.io/rootproject: "true" + component: argo-project + app: {{ .Chart.Name }} + release: {{ .Release.Name }} + chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.argocd.project.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + sourceRepos: + - '*' + sourceNamespaces: + - '*' + destinations: + - namespace: {{ .Values.argocd.namespace }} + server: https://kubernetes.default.svc + - namespace: {{ .Release.Name }} + server: https://kubernetes.default.svc + - namespace: "{{ .Release.Name }}-*" + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: '*' + kind: '*' + +# Generate an isolated argo project for each stage +{{ range $stage := .Values.stages }} +# Only output an argo project for this stage if the stage is enabled +{{- if or (not (hasKey $stage "enabled")) $stage.enabled }} +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: {{ $.Release.Name }}-{{ $stage.name }} + namespace: {{ $.Values.argocd.namespace }} + + annotations: + kargo-pipelines.ukserp.ac.uk/component: argo-stage-project + {{- with $.Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $.Values.argocd.project.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + + labels: + argocd.argoproj.io/rootproject: "true" + component: argo-stage-project + app: {{ $.Chart.Name }} + release: {{ $.Release.Name }} + chart: {{ printf "%s-%s" $.Chart.Name $.Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} + heritage: {{ $.Release.Service }} + {{- with $.Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $.Values.argocd.project.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + sourceRepos: + - '*' + sourceNamespaces: + - '*' + destinations: + - namespace: {{ $.Values.argocd.namespace }} + server: https://kubernetes.default.svc + - namespace: "{{ $.Release.Name }}-{{ $stage.name }}" + server: https://kubernetes.default.svc + - namespace: "{{ $.Release.Name }}-{{ $stage.name }}-*" + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: '*' + kind: '*' +{{- end }} +{{ end }} \ No newline at end of file diff --git a/charts/kargo-pipelines/templates/extra-manifests.yaml b/charts/kargo-pipelines/templates/extra-manifests.yaml new file mode 100644 index 0000000..b8336e5 --- /dev/null +++ b/charts/kargo-pipelines/templates/extra-manifests.yaml @@ -0,0 +1,6 @@ +{{- range $manifest := .Values.extraManifests }} +--- +{{- if typeIs "string" $manifest }} +{{ tpl $manifest $ }} +{{- end }} +{{- end }} diff --git a/charts/kargo-pipelines/templates/kargo/project.yaml b/charts/kargo-pipelines/templates/kargo/project.yaml new file mode 100644 index 0000000..9d83c00 --- /dev/null +++ b/charts/kargo-pipelines/templates/kargo/project.yaml @@ -0,0 +1,21 @@ +apiVersion: kargo.akuity.io/v1alpha1 +kind: Project +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + + annotations: + kargo-pipelines.ukserp.ac.uk/component: kargo-project + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + + labels: + component: kargo-project + app: {{ .Chart.Name }} + release: {{ .Release.Name }} + chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} diff --git a/charts/kargo-pipelines/templates/kargo/stage.yaml b/charts/kargo-pipelines/templates/kargo/stage.yaml new file mode 100644 index 0000000..5499d9f --- /dev/null +++ b/charts/kargo-pipelines/templates/kargo/stage.yaml @@ -0,0 +1,107 @@ +{{ range $stage := .Values.stages }} +--- +apiVersion: kargo.akuity.io/v1alpha1 +kind: Stage +metadata: + name: {{ $stage.name }} + namespace: {{ $.Release.Namespace }} + + annotations: + kargo-pipelines.ukserp.ac.uk/component: kargo-stage + {{- with $.Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $stage.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + + labels: + component: kargo-stage + app: {{ $.Chart.Name }} + release: {{ $.Release.Name }} + chart: {{ printf "%s-%s" $.Chart.Name $.Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} + heritage: {{ $.Release.Service }} + {{- with $.Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $stage.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + + requestedFreight: + - origin: + kind: Warehouse + name: {{ $.Release.Name }} + sources: + {{- with $stage.freightFrom }} + stages: {{ . | toJson }} + {{- else }} + direct: true + {{- end }} + + promotionTemplate: + spec: + steps: + + # Clone the sources for this stage, and prepare the deployment branch + - uses: git-clone + config: + repoURL: {{ $.Values.warehouse.valuesFreight.repoURL }} + checkout: + # Checkout the git commit from main referenced in this stages freight + # as the source directory + - fromFreight: true + fromOrigin: + kind: Warehouse + name: {{ $.Release.Name }} + path: ./src + + # Checkout the HEAD commit of the deployment branch for this stage + # as the output directory + - branch: deployment/{{ $.Release.Name }}-{{ $stage.name }} + create: true + path: ./out + + # Clear the deployment branch so the only files on it are live manifests + - uses: git-clear + config: + path: ./out + + # Fish into source manifests and update the image tags to match the freight + {{- range $.Values.warehouse.updateTemplates }} + - uses: yaml-update + as: {{ .name }} + config: + # Patch directly into the helm values contained within the airflow.yaml app manifest + path: ./src/{{ $.Values.warehouse.valuesFreight.path }}/{{ .path }} + updates: + {{- range .updates }} + - key: {{ .key }} + value: {{ .value }} + {{- end }} + {{ end }} + # Execute the kustomize for this stages deployment and output it into the + # empty deployment branch for this stage + - uses: kustomize-build + config: + path: ./src/{{ $.Values.warehouse.valuesFreight.path }}/deployments/{{ $stage.name }} + outPath: ./out + + # Commit the final state of the output directory to the deployment branch + - uses: git-commit + as: commit + config: + path: ./out + messageFromSteps: + - patch-applications + {{- range $.Values.warehouse.updateTemplates }} + - {{ .name }} + {{- end }} + + # Push the git commit to the deployment branch + - uses: git-push + config: + path: ./out + targetBranch: deployment/{{ $.Release.Name }}-{{ $stage.name }} +{{ end }} \ No newline at end of file diff --git a/charts/kargo-pipelines/templates/kargo/warehouse.yaml b/charts/kargo-pipelines/templates/kargo/warehouse.yaml new file mode 100644 index 0000000..9ffdd1f --- /dev/null +++ b/charts/kargo-pipelines/templates/kargo/warehouse.yaml @@ -0,0 +1,38 @@ +apiVersion: kargo.akuity.io/v1alpha1 +kind: Warehouse +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + + annotations: + kargo-pipelines.ukserp.ac.uk/component: kargo-warehouse + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.warehouse.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + + labels: + component: kargo-warehouse + app: {{ .Chart.Name }} + release: {{ .Release.Name }} + chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.warehouse.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + subscriptions: + + # Track the main branch of the deployment repo so that freight pins the + # deployment values and manifests + - git: + repoURL: {{ .Values.warehouse.valuesFreight.repoURL }} + discoveryLimit: {{ .Values.warehouse.valuesFreight.discoveryLimit }} + + # Track auxiliary freight that may need to be patched into manifests + {{- .Values.warehouse.freight | toYaml | nindent 4 -}} diff --git a/charts/kargo-pipelines/values.yaml b/charts/kargo-pipelines/values.yaml new file mode 100644 index 0000000..904ed75 --- /dev/null +++ b/charts/kargo-pipelines/values.yaml @@ -0,0 +1,76 @@ + +# Global annotations and labels applied to all manifests +annotations: {} +labels: {} + +# A list of strings that will be templated and then exported as individual manifests +extraManifests: [] + +namespace: + # Global annotations and labels applied to the stage namespaces + annotations: {} + labels: {} + +argocd: + # The top level argocd namespace to deploy the argo project and appset + namespace: argocd + + project: + # Annotations and labels applied to the project + annotations: {} + labels: {} + + app: + # Annotations and labels applied to the apps + annotations: {} + labels: {} + +# A list of stages to spawn and their dependencies +stages: [] +# - name: "dev-0" +# labels: +# example: label +# annotations: +# example: annotation +# - name: "dev-1" +# enabled: false +# - name: "uat" +# freightFrom: +# - "dev-0" +# - "dev-1" + +warehouse: + + # Annotations and labels applied to the warehouse + annotations: {} + labels: {} + + # Values freight is the commit on the main branch of the deployment repo which we want to deploy + valuesFreight: + repoURL: https://github.com/example/kargo-demo.git + discoveryLimit: 10 + # Path within the main branch of the repo to the kustomize project for this set of deployments + path: kargo-demo + + # Auxiliary freight are the image tags, chart versions, and git commits we want to expose for + # kargo to track and update. + freight: [] +# # Track a container image +# - image: +# repoURL: apache/airflow +# discoveryLimit: 10 +# # Track a chart version +# - chart: +# repoURL: https://airflow.apache.org +# name: airflow +# discoveryLimit: 10 + + # Define how to patch manifests before kustomization + updateTemplates: [] +# - name: patch-airflow +# path: base/manifests/airflow.yaml +# updates: +# - key: spec.source.helm.valuesObject.images.airflow.tag +# value: ${{ imageFrom("apache/airflow").tag }} +# - key: spec.source.targetRevision +# value: ${{ chartFrom("https://airflow.apache.org", "airflow").version }} diff --git a/example/kargo-demo/base/kustomization.yaml b/example/kargo-demo/base/kustomization.yaml new file mode 100644 index 0000000..65d5058 --- /dev/null +++ b/example/kargo-demo/base/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- manifests/airflow.yaml diff --git a/example/kargo-demo/base/manifests/airflow.yaml b/example/kargo-demo/base/manifests/airflow.yaml new file mode 100644 index 0000000..ac1f629 --- /dev/null +++ b/example/kargo-demo/base/manifests/airflow.yaml @@ -0,0 +1,67 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: airflow + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + # Put child apps in argo project for stage + project: PLACEHOLDER + destination: + server: https://kubernetes.default.svc + # Deploy to the stage namespace + namespace: PLACEHOLDER + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=false + source: + repoURL: 'https://airflow.apache.org' + chart: airflow + targetRevision: PLACEHOLDER + + helm: + # Use valuesObject so that we can fish into it with kargo and patch the freight + # values, and also apply kustomize patches to the values (ingress urls, ect) + valuesObject: + + executor: KubernetesExecutor + + images: + airflow: + repository: harbor.ukserp.ac.uk/dlm/airflow/dags + tag: PLACEHOLDER + pullPolicy: Always + + config: + core: + load_examples: "True" + + createUserJob: + useHelmHooks: false + applyCustomEnv: false + + migrateDatabaseJob: + useHelmHooks: false + applyCustomEnv: false + + webserver: + enabled: true + startupProbe: + timeoutSeconds: 30 + failureThreshold: 20 + periodSeconds: 30 + + triggerer: + enabled: true + + postgresql: + enabled: true + + pgbouncer: + enabled: true + maxClientConn: 1000 + metadataPoolSize: 100 + resultBackendPoolSize: 50 diff --git a/example/kargo-demo/deployments/dev-0/kustomization.yaml b/example/kargo-demo/deployments/dev-0/kustomization.yaml new file mode 100644 index 0000000..5098163 --- /dev/null +++ b/example/kargo-demo/deployments/dev-0/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base +- manifests/extra-configmap.yaml + +# TODO show example of patching the airflow values (ingress url) +patches: + - path: patches/applications.yaml + target: + group: argoproj.io + version: v1alpha1 + kind: Application diff --git a/example/kargo-demo/deployments/dev-0/manifests/extra-configmap.yaml b/example/kargo-demo/deployments/dev-0/manifests/extra-configmap.yaml new file mode 100644 index 0000000..f436ed8 --- /dev/null +++ b/example/kargo-demo/deployments/dev-0/manifests/extra-configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + # This is an extra config map only deployed in the dev-0 deployment + name: example-extra-configmap +data: + foo: "bar" diff --git a/example/kargo-demo/deployments/dev-0/patches/applications.yaml b/example/kargo-demo/deployments/dev-0/patches/applications.yaml new file mode 100644 index 0000000..dbe28f0 --- /dev/null +++ b/example/kargo-demo/deployments/dev-0/patches/applications.yaml @@ -0,0 +1,7 @@ +# TODO find a way to automate this patch +- op: replace + path: /spec/destination/namespace + value: kargo-demo-dev-0 +- op: replace + path: /spec/project + value: kargo-demo-dev-0 diff --git a/example/kargo-demo/deployments/dev-1/kustomization.yaml b/example/kargo-demo/deployments/dev-1/kustomization.yaml new file mode 100644 index 0000000..65f827b --- /dev/null +++ b/example/kargo-demo/deployments/dev-1/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base + +# TODO show example of patching the airflow values (ingress url) +patches: + - path: patches/applications.yaml + target: + group: argoproj.io + version: v1alpha1 + kind: Application diff --git a/example/kargo-demo/deployments/dev-1/patches/applications.yaml b/example/kargo-demo/deployments/dev-1/patches/applications.yaml new file mode 100644 index 0000000..a340fa4 --- /dev/null +++ b/example/kargo-demo/deployments/dev-1/patches/applications.yaml @@ -0,0 +1,7 @@ +# TODO find a way to automate this patch +- op: replace + path: /spec/destination/namespace + value: kargo-demo-dev-1 +- op: replace + path: /spec/project + value: kargo-demo-dev-1 diff --git a/example/kargo-demo/deployments/uat/kustomization.yaml b/example/kargo-demo/deployments/uat/kustomization.yaml new file mode 100644 index 0000000..65f827b --- /dev/null +++ b/example/kargo-demo/deployments/uat/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base + +# TODO show example of patching the airflow values (ingress url) +patches: + - path: patches/applications.yaml + target: + group: argoproj.io + version: v1alpha1 + kind: Application diff --git a/example/kargo-demo/deployments/uat/patches/applications.yaml b/example/kargo-demo/deployments/uat/patches/applications.yaml new file mode 100644 index 0000000..31494ea --- /dev/null +++ b/example/kargo-demo/deployments/uat/patches/applications.yaml @@ -0,0 +1,7 @@ +# TODO find a way to automate this patch +- op: replace + path: /spec/destination/namespace + value: kargo-demo-uat +- op: replace + path: /spec/project + value: kargo-demo-uat diff --git a/example/kargo-demo/kargo-pipeline.yaml b/example/kargo-demo/kargo-pipeline.yaml new file mode 100644 index 0000000..f0b6f2c --- /dev/null +++ b/example/kargo-demo/kargo-pipeline.yaml @@ -0,0 +1,84 @@ +# This application deploys the entire multi-stage stack of stacks... +# This example will deploy a namespace kargo-demo containing all the +# kargo and argo components needed to manage all the environments. +# +# For each stage requested an argo app kargo-demo-{{stage}} will be created. +# These apps will deploy into their own namespaces allowing them to be torn +# down independently. +# +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: kargo-demo + namespace: argocd + + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + # Put child apps in argo project for stage + project: default + destination: + server: https://kubernetes.default.svc + # Deploy to the stage namespace + namespace: kargo-demo + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + managedNamespaceMetadata: + labels: + kargo.akuity.io/project: "true" + source: + + # For testing, deploy off pinned harbor chart for actual projects + repoURL: https://github.com/SwanseaUniversityMedical/kargo-pipelines.git + path: charts/kargo-pipelines + targetRevision: HEAD + + helm: + valuesObject: + + # A list of stages to spawn and their dependencies + stages: + - name: "dev-0" + enabled: true + - name: "dev-1" + enabled: false + - name: "uat" + enabled: false + freightFrom: + - "dev-0" + - "dev-1" + + warehouse: + # Values freight is the commit on the main branch of the deployment repo which we want to deploy + valuesFreight: + repoURL: https://github.com/SwanseaUniversityMedical/kargo-pipelines.git + discoveryLimit: 10 + # Path within the main branch of the repo to the kustomize project for this set of deployments + path: example/kargo-demo + + # Auxiliary freight are the image tags, chart versions, and git commits we want to expose for + # kargo to track and update. + freight: + # Track a container image + - image: + repoURL: harbor.ukserp.ac.uk/dlm/airflow/dags + discoveryLimit: 10 + # Track a chart version + - chart: + repoURL: https://airflow.apache.org + name: airflow + discoveryLimit: 10 + + # Define how to patch manifests before kustomization + updateTemplates: + - name: patch-airflow + path: base/manifests/airflow.yaml + updates: + - key: spec.source.helm.valuesObject.images.airflow.tag + value: ${{ imageFrom("harbor.ukserp.ac.uk/dlm/airflow/dags").Tag }} + - key: spec.source.targetRevision + value: ${{ chartFrom("https://airflow.apache.org", "airflow").Version }}