add template parameters for support

Signed-off-by: Thibault Mange <[email protected]>
rhobs · Oct 3, 2023 · 83888ae · 83888ae
1 parent 7e42b25
commit 83888ae
Show file tree

Hide file tree

Showing 7 changed files with 169 additions and 86 deletions.
diff --git a/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-compact-template.yaml b/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-compact-template.yaml
@@ -33,8 +33,6 @@ objects:
       app.kubernetes.io/instance: observatorium
       app.kubernetes.io/name: thanos-compact
       app.kubernetes.io/part-of: observatorium
-  status:
-    loadBalancer: {}
 - apiVersion: v1
   kind: ServiceAccount
   metadata:
@@ -92,7 +90,7 @@ objects:
     name: observatorium-thanos-compact
     namespace: rhobs
   spec:
-    replicas: 1
+    replicas: ${THANOS_REPLICAS}
     selector:
       matchLabels:
         app.kubernetes.io/component: database-compactor
@@ -136,7 +134,7 @@ objects:
           - --delete-delay=24h0m0s
           - --downsample.concurrency=1
           - --log.format=logfmt
-          - --log.level=info
+          - --log.level=${THANOS_LOG_LEVEL}
           - --objstore.config=$(OBJSTORE_CONFIG)
           - --retention.resolution-1h=8760h0m0s
           - --retention.resolution-5m=8760h0m0s
@@ -322,3 +320,9 @@ parameters:
 - from: '[a-zA-Z0-9]{40}'
   generate: expression
   name: OAUTH_PROXY_COOKIE_SECRET
+- name: THANOS_LOG_LEVEL
+  required: true
+  value: info
+- name: THANOS_REPLICAS
+  required: true
+  value: "1"
diff --git a/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-store-template.yaml b/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-store-template.yaml
@@ -113,8 +113,6 @@ objects:
       app.kubernetes.io/instance: observatorium
       app.kubernetes.io/name: thanos-store
       app.kubernetes.io/part-of: observatorium
-  status:
-    loadBalancer: {}
 - apiVersion: v1
   kind: ServiceAccount
   metadata:
@@ -172,7 +170,7 @@ objects:
     name: observatorium-thanos-store
     namespace: rhobs
   spec:
-    replicas: 2
+    replicas: ${THANOS_REPLICAS}
     selector:
       matchLabels:
         app.kubernetes.io/component: object-store-gateway
@@ -213,7 +211,7 @@ objects:
           - --data-dir=/var/thanos/store
           - --ignore-deletion-marks-delay=24h0m0s
           - --log.format=logfmt
-          - --log.level=info
+          - --log.level=${THANOS_LOG_LEVEL}
           - --max-time=-22h0m0s
           - --objstore.config=$(OBJSTORE_CONFIG)
           - --selector.relabel-config-file=/tmp/config/hashmod-config.yaml
@@ -383,3 +381,10 @@ objects:
             storage: 500Gi
         storageClassName: gp2
       status: {}
+parameters:
+- name: THANOS_LOG_LEVEL
+  required: true
+  value: info
+- name: THANOS_REPLICAS
+  required: true
+  value: "2"
diff --git a/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-compact-template.yaml b/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-compact-template.yaml
@@ -33,8 +33,6 @@ objects:
       app.kubernetes.io/instance: observatorium
       app.kubernetes.io/name: thanos-compact
       app.kubernetes.io/part-of: observatorium
-  status:
-    loadBalancer: {}
 - apiVersion: v1
   kind: ServiceAccount
   metadata:
@@ -92,7 +90,7 @@ objects:
     name: observatorium-thanos-compact
     namespace: rhobs
   spec:
-    replicas: 1
+    replicas: ${THANOS_REPLICAS}
     selector:
       matchLabels:
         app.kubernetes.io/component: database-compactor
@@ -136,7 +134,7 @@ objects:
           - --delete-delay=24h0m0s
           - --downsample.concurrency=1
           - --log.format=logfmt
-          - --log.level=warn
+          - --log.level=${THANOS_LOG_LEVEL}
           - --objstore.config=$(OBJSTORE_CONFIG)
           - --retention.resolution-1h=8760h0m0s
           - --retention.resolution-5m=8760h0m0s
@@ -322,3 +320,9 @@ parameters:
 - from: '[a-zA-Z0-9]{40}'
   generate: expression
   name: OAUTH_PROXY_COOKIE_SECRET
+- name: THANOS_LOG_LEVEL
+  required: true
+  value: warn
+- name: THANOS_REPLICAS
+  required: true
+  value: "1"
diff --git a/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-store-template.yaml b/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-store-template.yaml
@@ -113,8 +113,6 @@ objects:
       app.kubernetes.io/instance: observatorium
       app.kubernetes.io/name: thanos-store
       app.kubernetes.io/part-of: observatorium
-  status:
-    loadBalancer: {}
 - apiVersion: v1
   kind: ServiceAccount
   metadata:
@@ -172,7 +170,7 @@ objects:
     name: observatorium-thanos-store
     namespace: rhobs
   spec:
-    replicas: 3
+    replicas: ${THANOS_REPLICAS}
     selector:
       matchLabels:
         app.kubernetes.io/component: object-store-gateway
@@ -213,7 +211,7 @@ objects:
           - --data-dir=/var/thanos/store
           - --ignore-deletion-marks-delay=24h0m0s
           - --log.format=logfmt
-          - --log.level=warn
+          - --log.level=${THANOS_LOG_LEVEL}
           - --max-time=-22h0m0s
           - --objstore.config=$(OBJSTORE_CONFIG)
           - --selector.relabel-config-file=/tmp/config/hashmod-config.yaml
@@ -383,3 +381,10 @@ objects:
             storage: 500Gi
         storageClassName: gp2
       status: {}
+parameters:
+- name: THANOS_LOG_LEVEL
+  required: true
+  value: warn
+- name: THANOS_REPLICAS
+  required: true
+  value: "3"
diff --git a/services_go/observatorium/metrics.go b/services_go/observatorium/metrics.go
@@ -6,15 +6,18 @@ import (
 	"maps"
 	"time"
 
+	"github.com/bwplotka/mimic/encoding"
 	"github.com/observatorium/observatorium/configuration_go/abstr/kubernetes/thanos/compactor"
 	"github.com/observatorium/observatorium/configuration_go/abstr/kubernetes/thanos/store"
 	"github.com/observatorium/observatorium/configuration_go/k8sutil"
+	"github.com/observatorium/observatorium/configuration_go/openshift"
 	"github.com/observatorium/observatorium/configuration_go/schemas/thanos/common"
 	"github.com/observatorium/observatorium/configuration_go/schemas/thanos/objstore"
 	objstore3 "github.com/observatorium/observatorium/configuration_go/schemas/thanos/objstore/s3"
 	trclient "github.com/observatorium/observatorium/configuration_go/schemas/thanos/tracing/client"
 	"github.com/observatorium/observatorium/configuration_go/schemas/thanos/tracing/jaeger"
 	routev1 "github.com/openshift/api/route/v1"
+	templatev1 "github.com/openshift/api/template/v1"
 	monv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	"gopkg.in/yaml.v3"
 	appsv1 "k8s.io/api/apps/v1"
@@ -37,7 +40,7 @@ const (
 var storeAutoShardRelabelConfigMap string
 
 // makeCompactor creates a base compactor component that can be derived from using the preManifestsHook.
-func makeCompactor(namespace, objstoreSecret string, preManifestsHook func(*compactor.CompactorStatefulSet)) k8sutil.ObjectMap {
+func makeCompactor(namespace, objstoreSecret string, preManifestsHook func(*compactor.CompactorStatefulSet)) encoding.Encoder {
 	// K8s config
 	compactorSatefulset := compactor.NewCompactor()
 	compactorSatefulset.Image = thanosImage
@@ -57,7 +60,6 @@ func makeCompactor(namespace, objstoreSecret string, preManifestsHook func(*comp
 	compactorSatefulset.Sidecars = []k8sutil.ContainerProvider{makeOauthProxy(10902, namespace, compactorSatefulset.Name, tlsSecret)}
 
 	// Compactor config
-	compactorSatefulset.Options.LogLevel = "warn"
 	compactorSatefulset.Options.RetentionResolutionRaw = 365 * 24 * time.Hour
 	compactorSatefulset.Options.RetentionResolution5m = 365 * 24 * time.Hour
 	compactorSatefulset.Options.RetentionResolution1h = 365 * 24 * time.Hour
@@ -67,10 +69,14 @@ func makeCompactor(namespace, objstoreSecret string, preManifestsHook func(*comp
 	compactorSatefulset.Options.DeduplicationReplicaLabel = "replica"
 	compactorSatefulset.Options.AddExtraOpts("--debug.max-compaction-level=3")
 
-	// Post process
+	// Execute preManifestsHook
 	if preManifestsHook != nil {
 		preManifestsHook(compactorSatefulset)
 	}
+	logLevel := string(compactorSatefulset.Options.LogLevel) // capture final log level for use in template
+	compactorSatefulset.Options.LogLevel = "${THANOS_LOG_LEVEL}"
+
+	// Post process
 	manifests := compactorSatefulset.Manifests()
 	service := getObject[*corev1.Service](manifests)
 	service.ObjectMeta.Annotations[servingCertSecretNameAnnotation] = tlsSecret
@@ -131,12 +137,38 @@ func makeCompactor(namespace, objstoreSecret string, preManifestsHook func(*comp
 		},
 	}
 
-	return manifests
+	// Wrap in template
+	template := openshift.WrapInTemplate("", manifests, metav1.ObjectMeta{
+		Name: "observatorium-metrics-compact",
+	}, []templatev1.Parameter{
+		{
+			Name:     "OAUTH_PROXY_COOKIE_SECRET",
+			Generate: "expression",
+			From:     "[a-zA-Z0-9]{40}",
+		},
+		{
+			Name:     "THANOS_LOG_LEVEL",
+			Value:    logLevel,
+			Required: true,
+		},
+		{
+			Name:     "THANOS_REPLICAS",
+			Value:    fmt.Sprintf("%d", compactorSatefulset.Replicas),
+			Required: true,
+		},
+	})
+
+	// Adding a special encoder wrapper to replace the replicas value in the template with a template parameter
+	// As the replicas value is typed as an int, it cannot be replaced using the compactor config.
+	yamlDecoder := templateYAML{encoder: encoding.GhodssYAML(template[""])}
+	yamlDecoder.AddReplacement(fmt.Sprintf(`(?m)^(\s*replicas: )%d$`, compactorSatefulset.Replicas), "${1}$${THANOS_REPLICAS}")
+
+	return &yamlDecoder
 
 }
 
 // makeStore creates a base store component that can be derived from using the preManifestsHook.
-func makeStore(namespace, objstoreSecret string, preManifestHook func(*store.StoreStatefulSet)) k8sutil.ObjectMap {
+func makeStore(namespace, objstoreSecret string, preManifestHook func(*store.StoreStatefulSet)) encoding.Encoder {
 	// K8s config
 	storeStatefulSet := store.NewStore()
 	storeStatefulSet.Image = thanosImage
@@ -207,10 +239,14 @@ func makeStore(namespace, objstoreSecret string, preManifestHook func(*store.Sto
 	}
 	storeStatefulSet.Options.StoreEnableIndexHeaderLazyReader = true // Enables parallel rolling update of store nodes.
 
-	// Post process
+	// Execute preManifestHook
 	if preManifestHook != nil {
 		preManifestHook(storeStatefulSet)
 	}
+	logLevel := string(storeStatefulSet.Options.LogLevel) // capture final log level for use in template
+	storeStatefulSet.Options.LogLevel = "${THANOS_LOG_LEVEL}"
+
+	// Post process
 	manifests := storeStatefulSet.Manifests()
 	postProcessServiceMonitor(getObject[*monv1.ServiceMonitor](manifests), storeStatefulSet.Namespace)
 	statefulset := getObject[*appsv1.StatefulSet](manifests)
@@ -307,7 +343,28 @@ func makeStore(namespace, objstoreSecret string, preManifestHook func(*store.Sto
 		},
 	}
 
-	return manifests
+	// Wrap in template
+	template := openshift.WrapInTemplate("", manifests, metav1.ObjectMeta{
+		Name: "observatorium-metrics-store",
+	}, []templatev1.Parameter{
+		{
+			Name:     "THANOS_LOG_LEVEL",
+			Value:    logLevel,
+			Required: true,
+		},
+		{
+			Name:     "THANOS_REPLICAS",
+			Value:    fmt.Sprintf("%d", storeStatefulSet.Replicas),
+			Required: true,
+		},
+	})
+
+	yamlDecoder := templateYAML{encoder: encoding.GhodssYAML(template[""])}
+	// Adding a special encoder wrapper to replace the replicas value in the template with a template parameter
+	// As the replicas value is typed as an int, it cannot be replaced using the compactor config.
+	yamlDecoder.AddReplacement(fmt.Sprintf(`(?m)^(\s*replicas: )%d$`, storeStatefulSet.Replicas), "${1}$${THANOS_REPLICAS}")
+
+	return &yamlDecoder
 }
 
 type kubeObject interface {

diff --git a/services_go/observatorium/observatorium.go b/services_go/observatorium/observatorium.go
@@ -3,19 +3,11 @@ package observatorium
 // import "github.com/rhobs/configuration/services_go/components/thanos/compactor"
 
 import (
-	"bytes"
-	"io"
-	"regexp"
-
 	"github.com/bwplotka/mimic"
 	"github.com/bwplotka/mimic/encoding"
 	"github.com/observatorium/api/rbac"
 	"github.com/observatorium/observatorium/configuration_go/abstr/kubernetes/thanos/compactor"
 	"github.com/observatorium/observatorium/configuration_go/abstr/kubernetes/thanos/store"
-	"github.com/observatorium/observatorium/configuration_go/k8sutil"
-	"github.com/observatorium/observatorium/configuration_go/openshift"
-	templatev1 "github.com/openshift/api/template/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // TenantInstanceConfiguration is the configuration for a single tenant in an instance of observatorium.
@@ -61,64 +53,13 @@ func NewObservatorium(cfg *InstanceConfiguration) *Observatorium {
 func (o *Observatorium) Manifests(generator *mimic.Generator) {
 	components := []struct {
 		name    string
-		objects k8sutil.ObjectMap
-		params  []templatev1.Parameter
+		encoder encoding.Encoder
 	}{
-		{"observatorium-metrics-compact", makeCompactor(o.cfg.Namespace, o.cfg.ObjStoreSecret, o.cfg.PreManifestsHooks.Compactor), []templatev1.Parameter{
-			{
-				Name:     "OAUTH_PROXY_COOKIE_SECRET",
-				Generate: "expression",
-				From:     "[a-zA-Z0-9]{40}",
-			},
-		}},
-		{"observatorium-metrics-store", makeStore(o.cfg.Namespace, o.cfg.ObjStoreSecret, o.cfg.PreManifestsHooks.ThanosStore), []templatev1.Parameter{}},
+		{"observatorium-metrics-compact", makeCompactor(o.cfg.Namespace, o.cfg.ObjStoreSecret, o.cfg.PreManifestsHooks.Compactor)},
+		{"observatorium-metrics-store", makeStore(o.cfg.Namespace, o.cfg.ObjStoreSecret, o.cfg.PreManifestsHooks.ThanosStore)},
 	}
 
 	for _, component := range components {
-		template := openshift.WrapInTemplate("", component.objects, metav1.ObjectMeta{
-			Name: component.name,
-		}, component.params)
-		generator.With(o.cfg.Cluster, o.cfg.Instance).Add(component.name+"-template.yaml", &customYAML{encoder: encoding.GhodssYAML(template[""])})
+		generator.With(o.cfg.Cluster, o.cfg.Instance).Add(component.name+"-template.yaml", &statusRemoveEncoder{encoder: component.encoder})
 	}
 }
-
-// customYAML is a YAML encoder wrapper that allows cleaning of the output.
-// Wihtout this, the manifests would contain a status section that is not needed.
-type customYAML struct {
-	encoder encoding.Encoder
-	reader  io.Reader
-}
-
-func (c *customYAML) Read(p []byte) (n int, err error) {
-	if c.reader == nil {
-		ret, err := io.ReadAll(c.encoder)
-		if err != nil {
-			panic(err)
-		}
-
-		c.reader = bytes.NewBuffer(c.clean(ret))
-	}
-
-	return c.reader.Read(p)
-}
-
-func (c *customYAML) EncodeComment(lines string) []byte {
-	return c.encoder.EncodeComment(lines)
-}
-
-func (c *customYAML) clean(input []byte) []byte {
-	// Remove status section from manifests
-	re := []*regexp.Regexp{
-		regexp.MustCompile(`\s*status:\n\s*availableReplicas: 0\n\s*replicas: 0`),
-		regexp.MustCompile(`\s*status:\n\s*currentHealthy: 0\n\s*desiredHealthy: 0\n\s*disruptionsAllowed: 0\n\s*expectedPods: 0`),
-		regexp.MustCompile(`\s*status:\n\s*ingress: null`),
-	}
-
-	ret := input
-
-	for _, r := range re {
-		ret = r.ReplaceAll(ret, []byte{})
-	}
-
-	return []byte(ret)
-}