wip

Signed-off-by: Thibault Mange <[email protected]>
rhobs · Oct 12, 2023 · d0b2c88 · d0b2c88
1 parent 8f892eb
commit d0b2c88
Show file tree

Hide file tree

Showing 11 changed files with 93 additions and 13 deletions.
diff --git a/...vices/app-sre-stage-01/rhobs/observatorium-metrics-receive-ingestor-default-template.yaml b/...vices/app-sre-stage-01/rhobs/observatorium-metrics-receive-ingestor-default-template.yaml
@@ -146,6 +146,12 @@ objects:
           - --log.level=${THANOS_LOG_LEVEL}
           - --objstore.config=$(OBJSTORE_CONFIG)
           - --remote-write.address=0.0.0.0:19291
+          - |
+            --tracing.config=type: JAEGER
+            config:
+              service_name: thanos-receive-router
+              sampler_type: ratelimiting
+              sampler_param: 2
           - --tsdb.path=/var/thanos/receive
           env:
           - name: OBJSTORE_CONFIG

diff --git a/...ervices/app-sre-stage-01/rhobs/observatorium-metrics-receive-router-default-template.yaml b/...ervices/app-sre-stage-01/rhobs/observatorium-metrics-receive-router-default-template.yaml
@@ -185,6 +185,25 @@ objects:
       observatorium/tenant: default
     name: observatorium-thanos-receive-controller-default
     namespace: rhobs
+- apiVersion: v1
+  data:
+    receive-limits.yaml: |
+      write:
+        default:
+          request:
+            samples_limit: 100000
+  kind: ConfigMap
+  metadata:
+    creationTimestamp: null
+    labels:
+      app.kubernetes.io/component: database-write-hashring-router
+      app.kubernetes.io/instance: observatorium
+      app.kubernetes.io/name: thanos-receive-router
+      app.kubernetes.io/part-of: observatorium
+      app.kubernetes.io/version: v0.32.4
+      observatorium/tenant: default
+    name: receive-limits-config-default
+    namespace: rhobs
 - apiVersion: apps/v1
   kind: Deployment
   metadata:
@@ -248,6 +267,12 @@ objects:
           - --receive.hashrings-file-refresh-interval=5s
           - --receive.limits-config-file=/etc/thanos/receive-limits/limits.yaml
           - --remote-write.address=0.0.0.0:19291
+          - |
+            --tracing.config=type: JAEGER
+            config:
+              service_name: thanos-receive-router
+              sampler_type: ratelimiting
+              sampler_param: 2
           image: quay.io/thanos/thanos:v0.32.4
           imagePullPolicy: IfNotPresent
           livenessProbe:
@@ -342,7 +367,7 @@ objects:
             name: thanos-receive-hashring-generated-default
           name: hashring-config
         - configMap:
-            name: receive-limits-config
+            name: receive-limits-config-default
           name: receive-limits-config
 - apiVersion: v1
   kind: Service

diff --git a/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-store-default-template.yaml b/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-store-default-template.yaml
@@ -229,6 +229,12 @@ objects:
           - --objstore.config=$(OBJSTORE_CONFIG)
           - --selector.relabel-config-file=/tmp/config/hashmod-config.yaml
           - --store.enable-index-header-lazy-reader=true
+          - |
+            --tracing.config=type: JAEGER
+            config:
+              service_name: thanos-store
+              sampler_type: ratelimiting
+              sampler_param: 2
           env:
           - name: HOST_IP_ADDRESS
             valueFrom:

diff --git a/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-store-rhel-template.yaml b/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-store-rhel-template.yaml
@@ -229,6 +229,12 @@ objects:
           - --objstore.config=$(OBJSTORE_CONFIG)
           - --selector.relabel-config-file=/tmp/config/hashmod-config.yaml
           - --store.enable-index-header-lazy-reader=true
+          - |
+            --tracing.config=type: JAEGER
+            config:
+              service_name: thanos-store
+              sampler_type: ratelimiting
+              sampler_param: 2
           env:
           - name: HOST_IP_ADDRESS
             valueFrom:

diff --git a/...urces/services/app-sre-stage-01/rhobs/observatorium-metrics-store-telemeter-template.yaml b/...urces/services/app-sre-stage-01/rhobs/observatorium-metrics-store-telemeter-template.yaml
@@ -229,6 +229,12 @@ objects:
           - --objstore.config=$(OBJSTORE_CONFIG)
           - --selector.relabel-config-file=/tmp/config/hashmod-config.yaml
           - --store.enable-index-header-lazy-reader=true
+          - |
+            --tracing.config=type: JAEGER
+            config:
+              service_name: thanos-store
+              sampler_type: ratelimiting
+              sampler_param: 2
           env:
           - name: HOST_IP_ADDRESS
             valueFrom:

diff --git a/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-store-default-template.yaml b/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-store-default-template.yaml
@@ -229,6 +229,12 @@ objects:
           - --objstore.config=$(OBJSTORE_CONFIG)
           - --selector.relabel-config-file=/tmp/config/hashmod-config.yaml
           - --store.enable-index-header-lazy-reader=true
+          - |
+            --tracing.config=type: JAEGER
+            config:
+              service_name: thanos-store
+              sampler_type: ratelimiting
+              sampler_param: 2
           env:
           - name: HOST_IP_ADDRESS
             valueFrom:

diff --git a/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-store-rhel-template.yaml b/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-store-rhel-template.yaml
@@ -229,6 +229,12 @@ objects:
           - --objstore.config=$(OBJSTORE_CONFIG)
           - --selector.relabel-config-file=/tmp/config/hashmod-config.yaml
           - --store.enable-index-header-lazy-reader=true
+          - |
+            --tracing.config=type: JAEGER
+            config:
+              service_name: thanos-store
+              sampler_type: ratelimiting
+              sampler_param: 2
           env:
           - name: HOST_IP_ADDRESS
             valueFrom:

diff --git a/...rces/services/telemeter-prod-01/rhobs/observatorium-metrics-store-telemeter-template.yaml b/...rces/services/telemeter-prod-01/rhobs/observatorium-metrics-store-telemeter-template.yaml
@@ -229,6 +229,12 @@ objects:
           - --objstore.config=$(OBJSTORE_CONFIG)
           - --selector.relabel-config-file=/tmp/config/hashmod-config.yaml
           - --store.enable-index-header-lazy-reader=true
+          - |
+            --tracing.config=type: JAEGER
+            config:
+              service_name: thanos-store
+              sampler_type: ratelimiting
+              sampler_param: 2
           env:
           - name: HOST_IP_ADDRESS
             valueFrom:

diff --git a/services_go/instances/rhobs/rhobs.go b/services_go/instances/rhobs/rhobs.go
@@ -47,7 +47,15 @@ func ClusterConfigs() []observatorium.Observatorium {
 					ThanosTenantConfig: observatorium.ThanosTenantConfig[receive.Router]{
 						Tenant: "default",
 					},
-					ReceiveLimits:             []struct{}{},
+					ReceiveLimits: &receive.ReceiveLimitsConfig{
+						WriteLimits: receive.WriteLimitsConfig{
+							DefaultLimits: receive.DefaultLimitsConfig{
+								RequestLimits: receive.RequestLimitsConfig{
+									SamplesLimit: 100000,
+								},
+							},
+						},
+					},
 					ReceiveControllerImageTag: "main-2023-09-22-f168dd7",
 				},
 			},

diff --git a/services_go/observatorium/metrics.go b/services_go/observatorium/metrics.go
@@ -12,9 +12,10 @@ import (
 	"github.com/observatorium/observatorium/configuration_go/abstr/kubernetes/thanos/store"
 	"github.com/observatorium/observatorium/configuration_go/k8sutil"
 	"github.com/observatorium/observatorium/configuration_go/openshift"
-	"github.com/observatorium/observatorium/configuration_go/schemas/thanos/common"
+	"github.com/observatorium/observatorium/configuration_go/schemas/thanos/log"
 	"github.com/observatorium/observatorium/configuration_go/schemas/thanos/objstore"
 	objstore3 "github.com/observatorium/observatorium/configuration_go/schemas/thanos/objstore/s3"
+	thanostime "github.com/observatorium/observatorium/configuration_go/schemas/thanos/time"
 	trclient "github.com/observatorium/observatorium/configuration_go/schemas/thanos/tracing/client"
 	"github.com/observatorium/observatorium/configuration_go/schemas/thanos/tracing/jaeger"
 	routev1 "github.com/openshift/api/route/v1"
@@ -54,10 +55,14 @@ func makeReceiveRouter(namespace, imageTag string, cfg ThanosReceiveRouterConfig
 	router.PodResources.Requests[corev1.ResourceMemory] = resource.MustParse("200Gi")
 	router.PodResources.Limits[corev1.ResourceMemory] = resource.MustParse("200Gi")
 	router.Sidecars = []k8sutil.ContainerProvider{makeJaegerAgent("observatorium-tools")}
+	receiveLimitsConfigMap := "receive-limits-config-" + cfg.Tenant
+	router.ConfigMaps[receiveLimitsConfigMap] = map[string]string{
+		"receive-limits.yaml": cfg.ReceiveLimits.String(),
+	}
 
 	// Router config
-	router.Options.LogLevel = common.LogLevelWarn
-	router.Options.LogFormat = common.LogFormatLogfmt
+	router.Options.LogLevel = log.LogLevelWarn
+	router.Options.LogFormat = log.LogFormatLogfmt
 	router.Options.TracingConfig = &trclient.TracingConfig{
 		Type: trclient.Jaeger,
 		Config: jaeger.Config{
@@ -81,7 +86,7 @@ func makeReceiveRouter(namespace, imageTag string, cfg ThanosReceiveRouterConfig
 	routerDeployment := getObject[*appsv1.Deployment](manifests)
 	routerDeployment.Spec.Template.Spec.Volumes = []corev1.Volume{
 		k8sutil.NewPodVolumeFromConfigMap("hashring-config", generatedHashringCm),
-		k8sutil.NewPodVolumeFromConfigMap("receive-limits-config", "receive-limits-config"),
+		k8sutil.NewPodVolumeFromConfigMap("receive-limits-config", receiveLimitsConfigMap),
 	}
 
 	// Add thanos-receive-controller
@@ -140,8 +145,8 @@ func makeReceiveIngestor(namespace, imageTag string, cfg ThanosTenantConfig[rece
 	ingestor.Sidecars = []k8sutil.ContainerProvider{makeJaegerAgent("observatorium-tools")}
 
 	// Router config
-	ingestor.Options.LogLevel = common.LogLevelWarn
-	ingestor.Options.LogFormat = common.LogFormatLogfmt
+	ingestor.Options.LogLevel = log.LogLevelWarn
+	ingestor.Options.LogFormat = log.LogFormatLogfmt
 	ingestor.Options.TracingConfig = &trclient.TracingConfig{
 		Type: trclient.Jaeger,
 		Config: jaeger.Config{
@@ -198,7 +203,7 @@ func makeCompactor(namespace, imageTag string, cfg ThanosTenantConfig[compactor.
 	compactorSatefulset.Sidecars = []k8sutil.ContainerProvider{makeOauthProxy(10902, namespace, compactorSatefulset.Name, tlsSecret)}
 
 	// Compactor config
-	compactorSatefulset.Options.LogLevel = common.LogLevelWarn
+	compactorSatefulset.Options.LogLevel = log.LogLevelWarn
 	compactorSatefulset.Options.RetentionResolutionRaw = 0
 	compactorSatefulset.Options.RetentionResolution5m = 0
 	compactorSatefulset.Options.RetentionResolution1h = 0
@@ -353,11 +358,11 @@ func makeStore(namespace, imageTag string, cfg ThanosTenantConfig[store.StoreSta
 	}
 
 	// Store config
-	storeStatefulSet.Options.LogLevel = common.LogLevelWarn
-	storeStatefulSet.Options.LogFormat = common.LogFormatLogfmt
+	storeStatefulSet.Options.LogLevel = log.LogLevelWarn
+	storeStatefulSet.Options.LogFormat = log.LogFormatLogfmt
 	storeStatefulSet.Options.IgnoreDeletionMarksDelay = 24 * time.Hour
 	maxTimeDur := time.Duration(-22) * time.Hour
-	storeStatefulSet.Options.MaxTime = &common.TimeOrDurationValue{Dur: &maxTimeDur}
+	storeStatefulSet.Options.MaxTime = &thanostime.TimeOrDurationValue{Dur: &maxTimeDur}
 	storeStatefulSet.Options.SelectorRelabelConfigFile = "/tmp/config/hashmod-config.yaml"
 	storeStatefulSet.Options.TracingConfig = &trclient.TracingConfig{
 		Type: trclient.Jaeger,

diff --git a/services_go/observatorium/observatorium.go b/services_go/observatorium/observatorium.go
@@ -25,7 +25,7 @@ type ThanosTenantConfig[T compactor.CompactorStatefulSet | store.StoreStatefulSe
 
 type ThanosReceiveRouterConfig struct {
 	ThanosTenantConfig[receive.Router]
-	ReceiveLimits             []struct{}
+	ReceiveLimits             *receive.ReceiveLimitsConfig
 	ReceiveControllerImageTag string
 }