feat: Add VPA for valkey, increase TTL and Git repository

- Mount data and config properly to valkey

.gitignore

@@ -17,3 +17,4 @@ certs/*
 /helm/charts/*
 /logs/*
 .env
+autoscaler

Makefile

@@ -152,6 +152,12 @@ create-cluster: registry
 	kubectl cluster-info
 	kubectl config set-context --current --namespace=$(NAMESPACE)
 
+install-vpa:
+	git clone https://github.com/kubernetes/autoscaler.git
+	cd autoscaler/vertical-pod-autoscaler
+	./hack/vpa-up.sh
+	kubectl --namespace=kube-system get pods | grep vpa
+
 delete-cluster:
 	k3d cluster list $(CLUSTER_NAME) 2>&- && k3d cluster delete $(CLUSTER_NAME)
 

backend/capellacollab/projects/toolmodels/modelsources/git/crud.py

@@ -63,12 +63,10 @@ def update_git_model(
     git_model: models.DatabaseGitModel,
     put_model: models.PutGitModel,
 ) -> models.DatabaseGitModel:
+    git_model.path = put_model.path
     git_model.entrypoint = put_model.entrypoint
     git_model.revision = put_model.revision
-
-    if put_model.path != git_model.path:
-        git_model.path = put_model.path
-        git_model.repository_id = None
+    git_model.repository_id = None
 
     if put_model.password:
         git_model.username = put_model.username

backend/capellacollab/projects/toolmodels/modelsources/git/handler/cache.py

@@ -5,6 +5,8 @@
 
 from capellacollab.core import database
 
+DEFAULT_TTL = datetime.timedelta(days=90)
+
 
 class GitValkeyCache:
     def __init__(self, git_model_id: int) -> None:
@@ -44,7 +46,6 @@ def put_file_data(
         last_updated: datetime.datetime,
         content: bytes,
         revision: str,
-        ttl: int = 3600,
     ) -> None:
         self._valkey.hset(
             name=self._get_file_key(file_path, revision),
@@ -54,7 +55,7 @@ def put_file_data(
             },
         )
         self._valkey.expire(
-            name=self._get_file_key(file_path, revision), time=ttl
+            name=self._get_file_key(file_path, revision), time=DEFAULT_TTL
         )
 
     def put_artifact_data(
@@ -63,14 +64,13 @@ def put_artifact_data(
         file_path: str,
         started_at: datetime.datetime,
         content: bytes,
-        ttl: int = 3600,
     ) -> None:
         self._valkey.hset(
             name=self._get_artifact_key(job_id, file_path),
             mapping={"started_at": started_at.isoformat(), "content": content},
         )
         self._valkey.expire(
-            name=self._get_artifact_key(job_id, file_path), time=ttl
+            name=self._get_artifact_key(job_id, file_path), time=DEFAULT_TTL
         )
 
     def clear(self) -> None:

backend/capellacollab/projects/toolmodels/modelsources/git/routes.py

@@ -144,6 +144,7 @@ def update_git_model_by_id(
     db: orm.Session = fastapi.Depends(database.get_db),
 ) -> models.DatabaseGitModel:
     git_util.verify_path_prefix(db, put_git_model.path)
+    cache.GitValkeyCache(git_model_id=db_git_model.id).clear()
     return crud.update_git_model(db, db_git_model, put_git_model)
 
 
@@ -175,5 +176,5 @@ def delete_git_model_by_id(
 ):
     if backups_crud.get_pipelines_for_git_model(db, db_git_model):
         raise exceptions.GitRepositoryUsedInPipelines(db_git_model.id)
-
+    cache.GitValkeyCache(git_model_id=db_git_model.id).clear()
     crud.delete_git_model(db, db_git_model)

backend/pyproject.toml

@@ -170,6 +170,7 @@ disable = [
   "too-few-public-methods",
   "too-many-ancestors",
   "too-many-arguments",
+  "too-many-positional-arguments",
   "too-many-boolean-expressions",
   "too-many-branches",
   "too-many-instance-attributes",

backend/tests/projects/toolmodels/conftest.py

@@ -40,7 +40,6 @@ def put_file_data(
             last_updated: datetime.datetime,
             content: bytes,
             revision: str,
-            ttl: int = 3600,
         ) -> None:
             MockGitValkeyCache.cache[f"f:{file_path}"] = (
                 last_updated,
@@ -53,7 +52,6 @@ def put_artifact_data(
             file_path: str,
             started_at: datetime.datetime,
             content: bytes,
-            ttl: int = 3600,
         ) -> None:
             MockGitValkeyCache.cache[f"a:{file_path}:{job_id}"] = (
                 started_at,

helm/config/backend.yaml

@@ -67,7 +67,7 @@ database:
   {{ end }}
 
 valkey:
-  url: "valkey://default:{{ .Values.valkey.password }}@{{ .Release.Name }}-backend-valkey:6379/0"
+  url: "valkey://default:{{ .Values.valkey.password }}@{{ .Release.Name }}-valkey:6379/0"
 
 pipelines:
   timeout: {{ .Values.pipelines.timeout }}

helm/templates/valkey/valkey.deployment.yaml

@@ -4,7 +4,7 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ .Release.Name }}-backend-valkey
+  name: {{ .Release.Name }}-valkey
   labels:
     id: {{ .Release.Name }}-deployment-valkey
 spec:
@@ -27,12 +27,13 @@ spec:
           secret:
             secretName: {{ .Release.Name }}-valkey
       containers:
-        - name: {{ .Release.Name }}-backend-valkey
+        - name: {{ .Release.Name }}-valkey
           {{ if .Values.docker.images.valkey }}
           image: {{ .Values.docker.images.valkey }}
           {{ else }}
           image: {{ .Values.docker.registry.external }}/valkey/valkey:7.2.6
           {{ end }}
+          args: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"]
           ports:
             - name: valkey
               containerPort: 6379
@@ -56,7 +57,8 @@ spec:
               ephemeral-storage: "5Gi"
             {{ end }}
           volumeMounts:
-            - mountPath: /valkey-master-data
+            - mountPath: /data
               name: data
-            - mountPath: /valkey-master
+            - mountPath: /usr/local/etc/valkey/valkey.conf
               name: config
+              subPath: valkey.conf

helm/templates/valkey/valkey.disruptionsbudget.yaml

@@ -4,7 +4,7 @@
 apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:
-  name: {{ .Release.Name }}-backend-valkey
+  name: {{ .Release.Name }}-valkey
 spec:
   minAvailable: 1
   selector:

helm/templates/valkey/valkey.service.yaml

@@ -4,9 +4,9 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ .Release.Name }}-backend-valkey
+  name: {{ .Release.Name }}-valkey
   labels:
-    id: {{ .Release.Name }}-service-backend-valkey
+    id: {{ .Release.Name }}-service-valkey
 spec:
   type: ClusterIP
   selector:

helm/templates/valkey/valkey.vpa.yaml

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
+# SPDX-License-Identifier: Apache-2.0
+
+{{ if .Values.cluster.vpa.enabled }}
+apiVersion: autoscaling.k8s.io/v1
+kind: VerticalPodAutoscaler
+metadata:
+  name: {{ .Release.Name }}-valkey
+  namespace: {{ .Release.Namespace }}
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ .Release.Name }}-valkey
+  updatePolicy:
+    updateMode: 'Off'
+{{ end }}

helm/values.yaml

@@ -243,6 +243,12 @@ cluster:
     service: kube-dns # dns-default for OpenShift
     namespace: kube-system # openshift-dns for OpenShift
 
+  # Enable Vertical Pod Autoscaler for resource recommandations
+  # If you want to enable it locally, install VPA first:
+  # `make install-vpa`
+  vpa:
+    enabled: false
+
 # Specify the NO_PROXY environment for the backend
 # Leave it empty if not needed
 proxy: