From 83888ae72a76e3297cc1f59d62e6ce1fe3fb0dfc Mon Sep 17 00:00:00 2001 From: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> Date: Tue, 3 Oct 2023 16:48:54 +0200 Subject: [PATCH] add template parameters for support Signed-off-by: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> --- ...bservatorium-metrics-compact-template.yaml | 12 ++-- .../observatorium-metrics-store-template.yaml | 13 ++-- ...bservatorium-metrics-compact-template.yaml | 12 ++-- .../observatorium-metrics-store-template.yaml | 13 ++-- services_go/observatorium/metrics.go | 71 +++++++++++++++++-- services_go/observatorium/observatorium.go | 67 ++--------------- services_go/observatorium/yaml.go | 67 +++++++++++++++++ 7 files changed, 169 insertions(+), 86 deletions(-) create mode 100644 services_go/observatorium/yaml.go diff --git a/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-compact-template.yaml b/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-compact-template.yaml index 325a133adf..3838675fa0 100755 --- a/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-compact-template.yaml +++ b/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-compact-template.yaml @@ -33,8 +33,6 @@ objects: app.kubernetes.io/instance: observatorium app.kubernetes.io/name: thanos-compact app.kubernetes.io/part-of: observatorium - status: - loadBalancer: {} - apiVersion: v1 kind: ServiceAccount metadata: @@ -92,7 +90,7 @@ objects: name: observatorium-thanos-compact namespace: rhobs spec: - replicas: 1 + replicas: ${THANOS_REPLICAS} selector: matchLabels: app.kubernetes.io/component: database-compactor @@ -136,7 +134,7 @@ objects: - --delete-delay=24h0m0s - --downsample.concurrency=1 - --log.format=logfmt - - --log.level=info + - --log.level=${THANOS_LOG_LEVEL} - --objstore.config=$(OBJSTORE_CONFIG) - --retention.resolution-1h=8760h0m0s - --retention.resolution-5m=8760h0m0s @@ -322,3 +320,9 @@ parameters: - from: '[a-zA-Z0-9]{40}' generate: expression name: OAUTH_PROXY_COOKIE_SECRET +- name: THANOS_LOG_LEVEL + required: true + value: info +- name: THANOS_REPLICAS + required: true + value: "1" diff --git a/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-store-template.yaml b/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-store-template.yaml index 402581e8dc..74804381ea 100755 --- a/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-store-template.yaml +++ b/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-store-template.yaml @@ -113,8 +113,6 @@ objects: app.kubernetes.io/instance: observatorium app.kubernetes.io/name: thanos-store app.kubernetes.io/part-of: observatorium - status: - loadBalancer: {} - apiVersion: v1 kind: ServiceAccount metadata: @@ -172,7 +170,7 @@ objects: name: observatorium-thanos-store namespace: rhobs spec: - replicas: 2 + replicas: ${THANOS_REPLICAS} selector: matchLabels: app.kubernetes.io/component: object-store-gateway @@ -213,7 +211,7 @@ objects: - --data-dir=/var/thanos/store - --ignore-deletion-marks-delay=24h0m0s - --log.format=logfmt - - --log.level=info + - --log.level=${THANOS_LOG_LEVEL} - --max-time=-22h0m0s - --objstore.config=$(OBJSTORE_CONFIG) - --selector.relabel-config-file=/tmp/config/hashmod-config.yaml @@ -383,3 +381,10 @@ objects: storage: 500Gi storageClassName: gp2 status: {} +parameters: +- name: THANOS_LOG_LEVEL + required: true + value: info +- name: THANOS_REPLICAS + required: true + value: "2" diff --git a/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-compact-template.yaml b/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-compact-template.yaml index 2b297a8805..564e1516b2 100755 --- a/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-compact-template.yaml +++ b/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-compact-template.yaml @@ -33,8 +33,6 @@ objects: app.kubernetes.io/instance: observatorium app.kubernetes.io/name: thanos-compact app.kubernetes.io/part-of: observatorium - status: - loadBalancer: {} - apiVersion: v1 kind: ServiceAccount metadata: @@ -92,7 +90,7 @@ objects: name: observatorium-thanos-compact namespace: rhobs spec: - replicas: 1 + replicas: ${THANOS_REPLICAS} selector: matchLabels: app.kubernetes.io/component: database-compactor @@ -136,7 +134,7 @@ objects: - --delete-delay=24h0m0s - --downsample.concurrency=1 - --log.format=logfmt - - --log.level=warn + - --log.level=${THANOS_LOG_LEVEL} - --objstore.config=$(OBJSTORE_CONFIG) - --retention.resolution-1h=8760h0m0s - --retention.resolution-5m=8760h0m0s @@ -322,3 +320,9 @@ parameters: - from: '[a-zA-Z0-9]{40}' generate: expression name: OAUTH_PROXY_COOKIE_SECRET +- name: THANOS_LOG_LEVEL + required: true + value: warn +- name: THANOS_REPLICAS + required: true + value: "1" diff --git a/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-store-template.yaml b/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-store-template.yaml index ab4d481da6..a3611e9bf7 100755 --- a/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-store-template.yaml +++ b/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-store-template.yaml @@ -113,8 +113,6 @@ objects: app.kubernetes.io/instance: observatorium app.kubernetes.io/name: thanos-store app.kubernetes.io/part-of: observatorium - status: - loadBalancer: {} - apiVersion: v1 kind: ServiceAccount metadata: @@ -172,7 +170,7 @@ objects: name: observatorium-thanos-store namespace: rhobs spec: - replicas: 3 + replicas: ${THANOS_REPLICAS} selector: matchLabels: app.kubernetes.io/component: object-store-gateway @@ -213,7 +211,7 @@ objects: - --data-dir=/var/thanos/store - --ignore-deletion-marks-delay=24h0m0s - --log.format=logfmt - - --log.level=warn + - --log.level=${THANOS_LOG_LEVEL} - --max-time=-22h0m0s - --objstore.config=$(OBJSTORE_CONFIG) - --selector.relabel-config-file=/tmp/config/hashmod-config.yaml @@ -383,3 +381,10 @@ objects: storage: 500Gi storageClassName: gp2 status: {} +parameters: +- name: THANOS_LOG_LEVEL + required: true + value: warn +- name: THANOS_REPLICAS + required: true + value: "3" diff --git a/services_go/observatorium/metrics.go b/services_go/observatorium/metrics.go index 4f516c64d8..914c082c3b 100644 --- a/services_go/observatorium/metrics.go +++ b/services_go/observatorium/metrics.go @@ -6,15 +6,18 @@ import ( "maps" "time" + "github.com/bwplotka/mimic/encoding" "github.com/observatorium/observatorium/configuration_go/abstr/kubernetes/thanos/compactor" "github.com/observatorium/observatorium/configuration_go/abstr/kubernetes/thanos/store" "github.com/observatorium/observatorium/configuration_go/k8sutil" + "github.com/observatorium/observatorium/configuration_go/openshift" "github.com/observatorium/observatorium/configuration_go/schemas/thanos/common" "github.com/observatorium/observatorium/configuration_go/schemas/thanos/objstore" objstore3 "github.com/observatorium/observatorium/configuration_go/schemas/thanos/objstore/s3" trclient "github.com/observatorium/observatorium/configuration_go/schemas/thanos/tracing/client" "github.com/observatorium/observatorium/configuration_go/schemas/thanos/tracing/jaeger" routev1 "github.com/openshift/api/route/v1" + templatev1 "github.com/openshift/api/template/v1" monv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "gopkg.in/yaml.v3" appsv1 "k8s.io/api/apps/v1" @@ -37,7 +40,7 @@ const ( var storeAutoShardRelabelConfigMap string // makeCompactor creates a base compactor component that can be derived from using the preManifestsHook. -func makeCompactor(namespace, objstoreSecret string, preManifestsHook func(*compactor.CompactorStatefulSet)) k8sutil.ObjectMap { +func makeCompactor(namespace, objstoreSecret string, preManifestsHook func(*compactor.CompactorStatefulSet)) encoding.Encoder { // K8s config compactorSatefulset := compactor.NewCompactor() compactorSatefulset.Image = thanosImage @@ -57,7 +60,6 @@ func makeCompactor(namespace, objstoreSecret string, preManifestsHook func(*comp compactorSatefulset.Sidecars = []k8sutil.ContainerProvider{makeOauthProxy(10902, namespace, compactorSatefulset.Name, tlsSecret)} // Compactor config - compactorSatefulset.Options.LogLevel = "warn" compactorSatefulset.Options.RetentionResolutionRaw = 365 * 24 * time.Hour compactorSatefulset.Options.RetentionResolution5m = 365 * 24 * time.Hour compactorSatefulset.Options.RetentionResolution1h = 365 * 24 * time.Hour @@ -67,10 +69,14 @@ func makeCompactor(namespace, objstoreSecret string, preManifestsHook func(*comp compactorSatefulset.Options.DeduplicationReplicaLabel = "replica" compactorSatefulset.Options.AddExtraOpts("--debug.max-compaction-level=3") - // Post process + // Execute preManifestsHook if preManifestsHook != nil { preManifestsHook(compactorSatefulset) } + logLevel := string(compactorSatefulset.Options.LogLevel) // capture final log level for use in template + compactorSatefulset.Options.LogLevel = "${THANOS_LOG_LEVEL}" + + // Post process manifests := compactorSatefulset.Manifests() service := getObject[*corev1.Service](manifests) service.ObjectMeta.Annotations[servingCertSecretNameAnnotation] = tlsSecret @@ -131,12 +137,38 @@ func makeCompactor(namespace, objstoreSecret string, preManifestsHook func(*comp }, } - return manifests + // Wrap in template + template := openshift.WrapInTemplate("", manifests, metav1.ObjectMeta{ + Name: "observatorium-metrics-compact", + }, []templatev1.Parameter{ + { + Name: "OAUTH_PROXY_COOKIE_SECRET", + Generate: "expression", + From: "[a-zA-Z0-9]{40}", + }, + { + Name: "THANOS_LOG_LEVEL", + Value: logLevel, + Required: true, + }, + { + Name: "THANOS_REPLICAS", + Value: fmt.Sprintf("%d", compactorSatefulset.Replicas), + Required: true, + }, + }) + + // Adding a special encoder wrapper to replace the replicas value in the template with a template parameter + // As the replicas value is typed as an int, it cannot be replaced using the compactor config. + yamlDecoder := templateYAML{encoder: encoding.GhodssYAML(template[""])} + yamlDecoder.AddReplacement(fmt.Sprintf(`(?m)^(\s*replicas: )%d$`, compactorSatefulset.Replicas), "${1}$${THANOS_REPLICAS}") + + return &yamlDecoder } // makeStore creates a base store component that can be derived from using the preManifestsHook. -func makeStore(namespace, objstoreSecret string, preManifestHook func(*store.StoreStatefulSet)) k8sutil.ObjectMap { +func makeStore(namespace, objstoreSecret string, preManifestHook func(*store.StoreStatefulSet)) encoding.Encoder { // K8s config storeStatefulSet := store.NewStore() storeStatefulSet.Image = thanosImage @@ -207,10 +239,14 @@ func makeStore(namespace, objstoreSecret string, preManifestHook func(*store.Sto } storeStatefulSet.Options.StoreEnableIndexHeaderLazyReader = true // Enables parallel rolling update of store nodes. - // Post process + // Execute preManifestHook if preManifestHook != nil { preManifestHook(storeStatefulSet) } + logLevel := string(storeStatefulSet.Options.LogLevel) // capture final log level for use in template + storeStatefulSet.Options.LogLevel = "${THANOS_LOG_LEVEL}" + + // Post process manifests := storeStatefulSet.Manifests() postProcessServiceMonitor(getObject[*monv1.ServiceMonitor](manifests), storeStatefulSet.Namespace) statefulset := getObject[*appsv1.StatefulSet](manifests) @@ -307,7 +343,28 @@ func makeStore(namespace, objstoreSecret string, preManifestHook func(*store.Sto }, } - return manifests + // Wrap in template + template := openshift.WrapInTemplate("", manifests, metav1.ObjectMeta{ + Name: "observatorium-metrics-store", + }, []templatev1.Parameter{ + { + Name: "THANOS_LOG_LEVEL", + Value: logLevel, + Required: true, + }, + { + Name: "THANOS_REPLICAS", + Value: fmt.Sprintf("%d", storeStatefulSet.Replicas), + Required: true, + }, + }) + + yamlDecoder := templateYAML{encoder: encoding.GhodssYAML(template[""])} + // Adding a special encoder wrapper to replace the replicas value in the template with a template parameter + // As the replicas value is typed as an int, it cannot be replaced using the compactor config. + yamlDecoder.AddReplacement(fmt.Sprintf(`(?m)^(\s*replicas: )%d$`, storeStatefulSet.Replicas), "${1}$${THANOS_REPLICAS}") + + return &yamlDecoder } type kubeObject interface { diff --git a/services_go/observatorium/observatorium.go b/services_go/observatorium/observatorium.go index 6b83b350b3..f7277aa439 100644 --- a/services_go/observatorium/observatorium.go +++ b/services_go/observatorium/observatorium.go @@ -3,19 +3,11 @@ package observatorium // import "github.com/rhobs/configuration/services_go/components/thanos/compactor" import ( - "bytes" - "io" - "regexp" - "github.com/bwplotka/mimic" "github.com/bwplotka/mimic/encoding" "github.com/observatorium/api/rbac" "github.com/observatorium/observatorium/configuration_go/abstr/kubernetes/thanos/compactor" "github.com/observatorium/observatorium/configuration_go/abstr/kubernetes/thanos/store" - "github.com/observatorium/observatorium/configuration_go/k8sutil" - "github.com/observatorium/observatorium/configuration_go/openshift" - templatev1 "github.com/openshift/api/template/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // TenantInstanceConfiguration is the configuration for a single tenant in an instance of observatorium. @@ -61,64 +53,13 @@ func NewObservatorium(cfg *InstanceConfiguration) *Observatorium { func (o *Observatorium) Manifests(generator *mimic.Generator) { components := []struct { name string - objects k8sutil.ObjectMap - params []templatev1.Parameter + encoder encoding.Encoder }{ - {"observatorium-metrics-compact", makeCompactor(o.cfg.Namespace, o.cfg.ObjStoreSecret, o.cfg.PreManifestsHooks.Compactor), []templatev1.Parameter{ - { - Name: "OAUTH_PROXY_COOKIE_SECRET", - Generate: "expression", - From: "[a-zA-Z0-9]{40}", - }, - }}, - {"observatorium-metrics-store", makeStore(o.cfg.Namespace, o.cfg.ObjStoreSecret, o.cfg.PreManifestsHooks.ThanosStore), []templatev1.Parameter{}}, + {"observatorium-metrics-compact", makeCompactor(o.cfg.Namespace, o.cfg.ObjStoreSecret, o.cfg.PreManifestsHooks.Compactor)}, + {"observatorium-metrics-store", makeStore(o.cfg.Namespace, o.cfg.ObjStoreSecret, o.cfg.PreManifestsHooks.ThanosStore)}, } for _, component := range components { - template := openshift.WrapInTemplate("", component.objects, metav1.ObjectMeta{ - Name: component.name, - }, component.params) - generator.With(o.cfg.Cluster, o.cfg.Instance).Add(component.name+"-template.yaml", &customYAML{encoder: encoding.GhodssYAML(template[""])}) + generator.With(o.cfg.Cluster, o.cfg.Instance).Add(component.name+"-template.yaml", &statusRemoveEncoder{encoder: component.encoder}) } } - -// customYAML is a YAML encoder wrapper that allows cleaning of the output. -// Wihtout this, the manifests would contain a status section that is not needed. -type customYAML struct { - encoder encoding.Encoder - reader io.Reader -} - -func (c *customYAML) Read(p []byte) (n int, err error) { - if c.reader == nil { - ret, err := io.ReadAll(c.encoder) - if err != nil { - panic(err) - } - - c.reader = bytes.NewBuffer(c.clean(ret)) - } - - return c.reader.Read(p) -} - -func (c *customYAML) EncodeComment(lines string) []byte { - return c.encoder.EncodeComment(lines) -} - -func (c *customYAML) clean(input []byte) []byte { - // Remove status section from manifests - re := []*regexp.Regexp{ - regexp.MustCompile(`\s*status:\n\s*availableReplicas: 0\n\s*replicas: 0`), - regexp.MustCompile(`\s*status:\n\s*currentHealthy: 0\n\s*desiredHealthy: 0\n\s*disruptionsAllowed: 0\n\s*expectedPods: 0`), - regexp.MustCompile(`\s*status:\n\s*ingress: null`), - } - - ret := input - - for _, r := range re { - ret = r.ReplaceAll(ret, []byte{}) - } - - return []byte(ret) -} diff --git a/services_go/observatorium/yaml.go b/services_go/observatorium/yaml.go new file mode 100644 index 0000000000..967c2ac60d --- /dev/null +++ b/services_go/observatorium/yaml.go @@ -0,0 +1,67 @@ +package observatorium + +import ( + "bytes" + "io" + "regexp" + + "github.com/bwplotka/mimic/encoding" +) + +// statusRemoveEncoder is a YAML encoder wrapper that allows cleaning of the output. +// Wihtout this, the manifests would contain a status section that is not needed. +type statusRemoveEncoder struct { + encoder encoding.Encoder + reader io.Reader +} + +func (c *statusRemoveEncoder) Read(p []byte) (n int, err error) { + if c.reader == nil { + ret, err := io.ReadAll(c.encoder) + if err != nil { + panic(err) + } + + // Remove status sections from manifests + c.reader = bytes.NewBuffer(regexp.MustCompile(`(?m)^( {2})status:\n( {4}.*\n)+`).ReplaceAll(ret, []byte{})) + } + + return c.reader.Read(p) +} + +func (c *statusRemoveEncoder) EncodeComment(lines string) []byte { + return c.encoder.EncodeComment(lines) +} + +// templateYAML is a YAML encoder wrapper that allows templating of the output. +// This is used when the target value is not typed as a string in Go. +type templateYAML struct { + encoder encoding.Encoder + reader io.Reader + replacements [][]string // regexp, replace tuples +} + +func (c *templateYAML) Read(p []byte) (n int, err error) { + if c.reader == nil { + ret, err := io.ReadAll(c.encoder) + if err != nil { + panic(err) + } + + for _, r := range c.replacements { + ret = regexp.MustCompile(r[0]).ReplaceAll(ret, []byte(r[1])) + } + + c.reader = bytes.NewBuffer(ret) + } + + return c.reader.Read(p) +} + +func (c *templateYAML) EncodeComment(lines string) []byte { + return c.encoder.EncodeComment(lines) +} + +func (c *templateYAML) AddReplacement(reg, replace string) { + c.replacements = append(c.replacements, []string{reg, replace}) +}