refactor!: Restructure SOPS files in Gitlab CI/CD templates

- The `unencrypted` suffix was removed from the files. Instead, it's recommended to use the `encrypted_regex` configuration as described in the admin documentation - The `general.values.yaml` and `secret.values.yaml` were merged into one encrypted file `values.yaml`. Also in this case it's recommended to use the `encrypted_regex` configuration. - Decrypt the sops file in-memory and never write them to disk. This improves security.
DSD-DBS · Jul 31, 2024 · c8c0c59 · c8c0c59
1 parent f8bfa76
commit c8c0c59
Show file tree

Hide file tree

Showing 4 changed files with 93 additions and 115 deletions.
diff --git a/ci-templates/gitlab/image-builder.yml b/ci-templates/gitlab/image-builder.yml
@@ -41,11 +41,10 @@ default:
   tags:
     - docker
 
-.decrypt: &decrypt
+.sops: &sops
   - apt-get install -y jq
   - gpg --import ${PRIVATE_GPG_PATH}
-  - sops -d ${TARGET}/secret.docker.json > plain.docker.json
-  - DOCKER_REGISTRY=$(cat plain.docker.json | jq -r ".registry_unencrypted")
+  - DOCKER_REGISTRY=$(sops -d ${TARGET}/secret.docker.json | jq -r ".registry")
 
 .github: &github
   - git clone https://github.com/DSD-DBS/capella-collab-manager.git
@@ -55,10 +54,10 @@ default:
 .docker: &docker
   - docker info
   - >
-    cat ../plain.docker.json | \
+    sops -d ../${TARGET}/secret.docker.json | \
       jq -r ".password" | \
       docker login \
-        -u $(cat ../plain.docker.json | jq -r ".username_unencrypted") \
+        -u $(sops -d ../${TARGET}/secret.docker.json | jq -r ".username") \
         --password-stdin \
         ${DOCKER_REGISTRY}
   - docker pull $BASE_IMAGE
@@ -72,7 +71,7 @@ default:
   - docker push ${IMAGE}:${DOCKER_TAG}
 
 .prepare: &prepare
-  - *decrypt
+  - *sops
   # prettier-ignore
   - DOCKER_TAG=$(echo ${REVISION} | sed 's/[^a-zA-Z0-9.]/-/g')-$CI_COMMIT_REF_SLUG
   - *github

diff --git a/ci-templates/gitlab/k8s-deploy.yml b/ci-templates/gitlab/k8s-deploy.yml
@@ -12,33 +12,12 @@ variables:
     value: 'main'
     description: 'Revision of the Github repository (capella-collab-manager)'
 
-.decrypt: &decrypt
+.sops: &sops
   - apt-get install -y jq
   - gpg --import ${PRIVATE_GPG_PATH}
-  - sops -d ${TARGET}/secret.values.yaml > plain.values.yaml
-  - sops -d ${TARGET}/secret.k8s.json > plain.k8s.json
-  # Remove _unencrypted suffix from plain.k8s.json
-  - |
-    python3 <<EOF
-    import json
-    import pathlib
-    import typing as t
-
-    def strip_suffix(data: t.Any):
-        if isinstance(data, dict):
-            return {key.replace('_unencrypted', ''): strip_suffix(value) for key, value in data.items()}
-        elif isinstance(data, list):
-            return [strip_suffix(element) for element in data]
-        else:
-            return data
-
-    path = pathlib.Path("plain.k8s.json")
-    data = json.loads(path.read_text())
-    path.write_text(json.dumps(strip_suffix(data), indent=4))
-    EOF
 
 .prepare: &prepare
-  - *decrypt
+  - *sops
   - git clone https://github.com/DSD-DBS/capella-collab-manager
   - cd capella-collab-manager
   - git checkout ${REVISION}
@@ -49,22 +28,22 @@ variables:
     ./helm/Chart.yaml
 
 .kubernetes: &kubernetes
-  - NAMESPACE=$(cat ../plain.k8s.json | jq -r ".namespace")
+  - NAMESPACE=$(sops -d ../${TARGET}/secret.k8s.json | jq -r ".namespace")
   # prettier-ignore
-  - kubectl config set-cluster cluster --server=$(cat ../plain.k8s.json | jq -r ".server")
+  - kubectl config set-cluster cluster --server=$(sops -d ../${TARGET}/secret.k8s.json | jq -r ".server")
   - >
     kubectl config set-context context \
       --namespace=${NAMESPACE} \
-      --user=$(cat ../plain.k8s.json | jq -r ".username") \
+      --user=$(sops -d ../${TARGET}/secret.k8s.json | jq -r ".username") \
       --cluster=cluster
   - >
     kubectl config set-credentials \
-      $(cat ../plain.k8s.json | jq -r ".username") \
-      --token=$(cat ../plain.k8s.json | jq -r ".token")
+      $(sops -d ../${TARGET}/secret.k8s.json | jq -r ".username") \
+      --token=$(sops -d ../${TARGET}/secret.k8s.json | jq -r ".token")
   - kubectl config use-context context
 
 .helm-deploy: &helm-deploy
-  - RELEASE=$(cat ../plain.k8s.json | jq -r ".release")
+  - RELEASE=$(sops -d ../${TARGET}/secret.k8s.json | jq -r ".release")
   - cp -r ../config/* helm/config
   # prettier-ignore
   - DOCKER_TAG=$(echo $REVISION | sed 's/[^a-zA-Z0-9.]/-/g')-$CI_COMMIT_REF_SLUG
@@ -76,12 +55,13 @@ variables:
       --version="$(git describe --tags)" \
       -d "$HELM_PACKAGE_DIR" \
       helm
-  - >
+  - |
+    sops -d ../${TARGET}/values.yaml | \
     helm upgrade ${RELEASE} \
       --namespace ${NAMESPACE} \
       --set docker.tag=${DOCKER_TAG} \
-      -f ../${TARGET}/general.values.yaml \
-      -f ../plain.values.yaml "$HELM_PACKAGE_DIR"/collab-manager-*.tgz
+      -f - \
+      "$HELM_PACKAGE_DIR"/collab-manager-*.tgz
 
   - kubectl rollout restart deployment ${RELEASE}-backend
   - kubectl rollout restart deployment ${RELEASE}-frontend

diff --git a/docs/docs/admin/ci-templates/gitlab/image-builder.md b/docs/docs/admin/ci-templates/gitlab/image-builder.md
@@ -5,13 +5,8 @@
 
 # Image Builder (Gitlab CI/CD)
 
-The image builder template builds the following images and pushes them to any
-Docker registry:
-
-- backend
-- frontend
-- documentation
-- guacamole
+The image builder template builds the images and pushes them to any Docker
+registry.
 
 Please add the following section to your `.gitlab-ci.yml`:
 
@@ -21,29 +16,19 @@ include:
 ```
 
 The build images are tagged with the revision they were build with (e.g., when
-running for main the tag would be `:main`) All characters matching the regex
+running for main the tag would be `:main`). All characters matching the regex
 [^a-za-z0-9.] will be replaced with -.
 
 You have to add the following environment variables on repository level. Make
 sure to enable the "Expand variable reference" flag.
 
 - `PRIVATE_GPG_PATH`: Path to the private GPG key used to decrypt the
   `secret.docker.json` file (More about this file [below](#docker-sops-file))
-- Variables speciying how to name each image:
-  - `FRONTEND_IMAGE_NAME` (defaults to _capella/collab/frontend_)
-  - `BACKEND_IMAGE_NAME` (default to _capella/collab/backend_)
-  - `DOCS_IMAGE_NAME` (defaults to _capella/collab/docs_)
-  - `GUACAMOLE_IMAGE_NAME` (defaults to _capella/collab/guacamole_)
-
-In addition you can adjust the following variables when running a pipeline:
-
-- Variables specifying whether to build an image (default to 1):
-  - `FRONTEND`: Build the frontend image?
-  - `BACKEND`: Build the backend image?
-  - `DOCS`: Build the docs image?
-  - `GUACAMOLE`: Build the guacamole image?
-- `TARGET`: The target for which you want to build the images (More information
-  why this is important [below](#docker-sops-file))
+- Variables specifying how to name each image:
+  - `FRONTEND_IMAGE_NAME` (defaults to `capella/collab/frontend`)
+  - `BACKEND_IMAGE_NAME` (default to `capella/collab/backend`)
+  - `DOCS_IMAGE_NAME` (defaults to `capella/collab/docs`)
+  - `GUACAMOLE_IMAGE_NAME` (defaults to `capella/collab/guacamole`)
 
 This is the (minimal) configuration. For more advanced configuration options,
 please refer to the
@@ -52,43 +37,41 @@ Gitlab template.
 
 ### Docker SOPS File
 
-We make use of [Mozilla SOPS](https://github.com/mozilla/sops) files to store
-secrets used in the image builder template. Therefore you need to have a
-directory `$TARGET` for each target with a `secret.docker.json` inside. You can
-create the `secret.docker.json` by running the following command:
-
-```zsh
-sops -e --output ./<target>/secret.docker.json input.json
-```
-
-The `input.json` in this command is a placeholder for your own input file,
-which should have the following structure:
-
-```json
-{
-  "registry_unencrypted": "<registry>",
-  "username_unencrypted": "<username>",
-  "password": "<unencrypted password>"
-}
-```
-
-Verify that you can open the secret file with
-`sops ./<target>/secret.docker.json`. When it works, delete the `input.json`!
+We make use of [SOPS](https://github.com/getsops/sops) files to store secrets
+used in the image builder template.
 
-In addition, you will need a `.sops.yaml` at the root level having the
+Create a file `.sops.yaml` at the root level of the repository with the
 following structure:
 
 ```yaml
 creation_rules:
   - path_regex: .*
+    encrypted_regex: ^(password|secret|adminPassword|uri|token)
     key_groups:
       - pgp:
           - <GPG fingerprint>
 ```
 
-Any time you update the `.sops.yaml` (i.e., adding or removing a fingerprint)
-you will have to run `sops updatekeys ./<target>/secret.docker.json` to ensure
-that only authorized persons can decrypt the secret file.
+Ensure that the GPG fingerprint of the Gitlab runner is present in the
+`.sops.yaml` such that it can decrypt the file.
 
-Lastly, please ensure that your Gitlab runners GPG fingerprint is present in
-the `.sops.yaml` such that it can use the secret values.
+You need to have a directory `$TARGET` for each target with a
+`secret.docker.json` inside. You can create the `secret.docker.json` by running
+the following command:
+
+```zsh
+sops edit ./$TARGET/secret.docker.json
+```
+
+Then, enter the following content:
+
+```json
+{
+  "registry": "<registry>",
+  "username": "<username>",
+  "password": "<password>"
+}
+```
+
+Verify that you can open the secret file with
+`sops ./<target>/secret.docker.json`.
diff --git a/docs/docs/admin/ci-templates/gitlab/k8s-deploy.md b/docs/docs/admin/ci-templates/gitlab/k8s-deploy.md
@@ -24,38 +24,57 @@ sure to enable the "Expand variable reference" flag.
   the Grafana Helm chart. It is useful if your deployment environment has
   limited access, so you can specify a URL that is accessible for you.
 
-In addition you can adjust the following variables when running a pipeline:
+## SOPS configuration
 
-- `TARGET`: The target for which you want to build the images (More information
-  why this is important [below](#docker-and-kubernetes-sops-files))
-- `REVISION`: The revision of the capella collab manager repository you want to
-  use
+We make use of [SOPS](https://github.com/getsops/sops) files to store secrets
+used in the deployment template.
 
-### Docker and Kubernetes SOPS Files
+Create a file `.sops.yaml` at the root level of the repository with the
+following structure:
 
-For the `k8s-deploy.yml` you need to have some secret sops files in place.
-First of all, you need a `secret.docker.json` file as described
-[here](#docker-sops-file). In addition, you need to have a `secret.k8s.json` in
-each target directory created by a json file having the following structure:
+```yaml
+creation_rules:
+  - path_regex: .*
+    encrypted_regex: ^(password|secret|adminPassword|uri|token)
+    key_groups:
+      - pgp:
+          - <GPG fingerprint>
+```
+
+Ensure that the GPG fingerprint of the Gitlab runner is present in the
+`.sops.yaml` such that it can decrypt the file.
+
+---
+
+Create a file to store the Kubernetes configuration:
+
+```zsh
+sops edit ./$TARGET/secret.k8s.json
+```
+
+The file has to contain the following content:
 
 ```json
 {
-  "server_unencrypted": "<k8s server>",
-  "namespace_unencrypted": "<namespace>",
-  "release_unencrypted": "<release>",
-  "username_unencrypted": "<username>",
+  "server": "<k8s server>",
+  "namespace": "<namespace>",
+  "release": "<release>",
+  "username": "<username>",
   "token": "<unencrypted token>"
 }
 ```
 
-In addition, you need to have a `general.values.yaml` containing all the
-`values.yaml` values that do not have to be encrypted and a
-`secret.values.yaml` only containing the values that should be encrypted
-(Please do not use YAML anchors in the `secret.values.yaml` file and do not use
-the `_unencrypted` suffix in the variable names).
+---
+
+Another configuration file is the encrypted `values.yaml`. In this file you can
+overwrite values from the
+[default `values.yaml`](https://github.com/DSD-DBS/capella-collab-manager/blob/main/helm/values.yaml).
 
-Please delete the plain text files containing secrets directly after encrypting
-them.
+Create the file with:
+
+```zsh
+sops edit ./$TARGET/values.yaml
+```
 
 ## Gitlab Repository Tree
 
@@ -64,15 +83,12 @@ The tree inside of your Gitlab repository should look like:
 ```zsh
 ├── .gitlab-ci.yml
 ├── .sops.yaml
-├── favicon.ico
 ├── target1
-│   ├── general.values.yaml
-│   ├── secret.values.yaml
+│   ├── values.yaml
 │   ├── secret.docker.json
 │   └── secret.k8s.json
 ├── target2
-│   ├── general.values.yaml
-│   ├── secret.values.yaml
+│   ├── values.yaml
 │   ├── secret.docker.json
 │   └── secret.k8s.json
 └── ...