diff --git a/resources/services/app-sre-stage-01/rhobs/default/observatorium-metrics-receive-ingestor-default-template.yaml b/resources/services/app-sre-stage-01/rhobs/default/observatorium-metrics-receive-ingestor-default-template.yaml index f0f430440f..5d1df0abd7 100755 --- a/resources/services/app-sre-stage-01/rhobs/default/observatorium-metrics-receive-ingestor-default-template.yaml +++ b/resources/services/app-sre-stage-01/rhobs/default/observatorium-metrics-receive-ingestor-default-template.yaml @@ -144,8 +144,7 @@ objects: - receive - --grpc-address=0.0.0.0:10901 - --http-address=0.0.0.0:10902 - - --label=receive=true - - --label=receive-replica=$(NAME) + - --label=replica="$(POD_NAME)" - --log.format=logfmt - --log.level=${LOG_LEVEL} - --objstore.config=$(OBJSTORE_CONFIG) @@ -190,6 +189,10 @@ objects: bucket: $(OBJ_STORE_BUCKET) endpoint: $(OBJ_STORE_ENDPOINT) region: $(OBJ_STORE_REGION) + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name image: quay.io/thanos/thanos:v0.32.4 imagePullPolicy: IfNotPresent livenessProbe: diff --git a/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-receive-router-template.yaml b/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-receive-router-template.yaml index 96f85504c0..435f1ab98d 100755 --- a/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-receive-router-template.yaml +++ b/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-receive-router-template.yaml @@ -240,6 +240,7 @@ objects: - receive - --grpc-address=0.0.0.0:10901 - --http-address=0.0.0.0:10902 + - --label=receive="true" - --log.format=logfmt - --log.level=${LOG_LEVEL} - --receive.hashrings-algorithm=ketama diff --git a/resources/services/app-sre-stage-01/rhobs/rhel/observatorium-metrics-receive-ingestor-rhel-template.yaml b/resources/services/app-sre-stage-01/rhobs/rhel/observatorium-metrics-receive-ingestor-rhel-template.yaml index 5ed6858f74..4bf57ca99b 100755 --- a/resources/services/app-sre-stage-01/rhobs/rhel/observatorium-metrics-receive-ingestor-rhel-template.yaml +++ b/resources/services/app-sre-stage-01/rhobs/rhel/observatorium-metrics-receive-ingestor-rhel-template.yaml @@ -144,8 +144,7 @@ objects: - receive - --grpc-address=0.0.0.0:10901 - --http-address=0.0.0.0:10902 - - --label=receive=true - - --label=receive-replica=$(NAME) + - --label=replica="$(POD_NAME)" - --log.format=logfmt - --log.level=${LOG_LEVEL} - --objstore.config=$(OBJSTORE_CONFIG) @@ -190,6 +189,10 @@ objects: bucket: $(OBJ_STORE_BUCKET) endpoint: $(OBJ_STORE_ENDPOINT) region: $(OBJ_STORE_REGION) + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name image: quay.io/thanos/thanos:v0.32.4 imagePullPolicy: IfNotPresent livenessProbe: diff --git a/resources/services/app-sre-stage-01/rhobs/telemeter/observatorium-metrics-receive-ingestor-telemeter-template.yaml b/resources/services/app-sre-stage-01/rhobs/telemeter/observatorium-metrics-receive-ingestor-telemeter-template.yaml index abcfa21695..16488b517d 100755 --- a/resources/services/app-sre-stage-01/rhobs/telemeter/observatorium-metrics-receive-ingestor-telemeter-template.yaml +++ b/resources/services/app-sre-stage-01/rhobs/telemeter/observatorium-metrics-receive-ingestor-telemeter-template.yaml @@ -144,8 +144,7 @@ objects: - receive - --grpc-address=0.0.0.0:10901 - --http-address=0.0.0.0:10902 - - --label=receive=true - - --label=receive-replica=$(NAME) + - --label=replica="$(POD_NAME)" - --log.format=logfmt - --log.level=${LOG_LEVEL} - --objstore.config=$(OBJSTORE_CONFIG) @@ -190,6 +189,10 @@ objects: bucket: $(OBJ_STORE_BUCKET) endpoint: $(OBJ_STORE_ENDPOINT) region: $(OBJ_STORE_REGION) + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name image: quay.io/thanos/thanos:v0.32.4 imagePullPolicy: IfNotPresent livenessProbe: diff --git a/resources/services/telemeter-prod-01/rhobs/default/observatorium-metrics-receive-ingestor-default-template.yaml b/resources/services/telemeter-prod-01/rhobs/default/observatorium-metrics-receive-ingestor-default-template.yaml index f0f430440f..5d1df0abd7 100755 --- a/resources/services/telemeter-prod-01/rhobs/default/observatorium-metrics-receive-ingestor-default-template.yaml +++ b/resources/services/telemeter-prod-01/rhobs/default/observatorium-metrics-receive-ingestor-default-template.yaml @@ -144,8 +144,7 @@ objects: - receive - --grpc-address=0.0.0.0:10901 - --http-address=0.0.0.0:10902 - - --label=receive=true - - --label=receive-replica=$(NAME) + - --label=replica="$(POD_NAME)" - --log.format=logfmt - --log.level=${LOG_LEVEL} - --objstore.config=$(OBJSTORE_CONFIG) @@ -190,6 +189,10 @@ objects: bucket: $(OBJ_STORE_BUCKET) endpoint: $(OBJ_STORE_ENDPOINT) region: $(OBJ_STORE_REGION) + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name image: quay.io/thanos/thanos:v0.32.4 imagePullPolicy: IfNotPresent livenessProbe: diff --git a/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-receive-router-template.yaml b/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-receive-router-template.yaml index dcb9afcaed..591bfba35b 100755 --- a/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-receive-router-template.yaml +++ b/resources/services/telemeter-prod-01/rhobs/observatorium-metrics-receive-router-template.yaml @@ -235,6 +235,7 @@ objects: - receive - --grpc-address=0.0.0.0:10901 - --http-address=0.0.0.0:10902 + - --label=receive="true" - --log.format=logfmt - --log.level=${LOG_LEVEL} - --receive.hashrings-algorithm=ketama diff --git a/resources/services/telemeter-prod-01/rhobs/rhel/observatorium-metrics-receive-ingestor-rhel-template.yaml b/resources/services/telemeter-prod-01/rhobs/rhel/observatorium-metrics-receive-ingestor-rhel-template.yaml index 5ed6858f74..4bf57ca99b 100755 --- a/resources/services/telemeter-prod-01/rhobs/rhel/observatorium-metrics-receive-ingestor-rhel-template.yaml +++ b/resources/services/telemeter-prod-01/rhobs/rhel/observatorium-metrics-receive-ingestor-rhel-template.yaml @@ -144,8 +144,7 @@ objects: - receive - --grpc-address=0.0.0.0:10901 - --http-address=0.0.0.0:10902 - - --label=receive=true - - --label=receive-replica=$(NAME) + - --label=replica="$(POD_NAME)" - --log.format=logfmt - --log.level=${LOG_LEVEL} - --objstore.config=$(OBJSTORE_CONFIG) @@ -190,6 +189,10 @@ objects: bucket: $(OBJ_STORE_BUCKET) endpoint: $(OBJ_STORE_ENDPOINT) region: $(OBJ_STORE_REGION) + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name image: quay.io/thanos/thanos:v0.32.4 imagePullPolicy: IfNotPresent livenessProbe: diff --git a/resources/services/telemeter-prod-01/rhobs/telemeter/observatorium-metrics-receive-ingestor-telemeter-template.yaml b/resources/services/telemeter-prod-01/rhobs/telemeter/observatorium-metrics-receive-ingestor-telemeter-template.yaml index abcfa21695..16488b517d 100755 --- a/resources/services/telemeter-prod-01/rhobs/telemeter/observatorium-metrics-receive-ingestor-telemeter-template.yaml +++ b/resources/services/telemeter-prod-01/rhobs/telemeter/observatorium-metrics-receive-ingestor-telemeter-template.yaml @@ -144,8 +144,7 @@ objects: - receive - --grpc-address=0.0.0.0:10901 - --http-address=0.0.0.0:10902 - - --label=receive=true - - --label=receive-replica=$(NAME) + - --label=replica="$(POD_NAME)" - --log.format=logfmt - --log.level=${LOG_LEVEL} - --objstore.config=$(OBJSTORE_CONFIG) @@ -190,6 +189,10 @@ objects: bucket: $(OBJ_STORE_BUCKET) endpoint: $(OBJ_STORE_ENDPOINT) region: $(OBJ_STORE_REGION) + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name image: quay.io/thanos/thanos:v0.32.4 imagePullPolicy: IfNotPresent livenessProbe: diff --git a/services_go/observatorium/metrics.go b/services_go/observatorium/metrics.go index 83ecf21583..a8d0533fa5 100644 --- a/services_go/observatorium/metrics.go +++ b/services_go/observatorium/metrics.go @@ -129,6 +129,12 @@ func (o ObservatoriumMetrics) makeReceiveRouter() encoding.Encoder { ServiceName: "thanos-receive-router", }, } + router.Options.Label = []receive.Label{ + { + Key: "receive", + Value: "\"true\"", + }, + } receiveLimits := receive.NewReceiveLimitsConfig() receiveLimits.WriteLimits.DefaultLimits = o.ReceiveLimitsDefault @@ -253,6 +259,7 @@ func (o ObservatoriumMetrics) makeTenantReceiveIngestor(instanceCfg *Observatori ingestor.PodResources.Limits[corev1.ResourceMemory] = resource.MustParse("24Gi") ingestor.Env = deleteObjStoreEnv(ingestor.Env) // delete the default objstore env vars ingestor.Env = append(ingestor.Env, objStoreEnvVars(instanceCfg.ObjStoreSecret)...) + ingestor.Env = append(ingestor.Env, k8sutil.NewEnvFromField("POD_NAME", "metadata.name")) ingestor.Sidecars = []k8sutil.ContainerProvider{makeJaegerAgent("observatorium-tools")} // Router config @@ -266,6 +273,12 @@ func (o ObservatoriumMetrics) makeTenantReceiveIngestor(instanceCfg *Observatori ServiceName: "thanos-receive-router", }, } + ingestor.Options.Label = []receive.Label{ + { + Key: "replica", + Value: "\"$(POD_NAME)\"", + }, + } // Execute preManifestsHook if instanceCfg.ReceiveIngestorPreManifestsHook != nil { @@ -757,13 +770,21 @@ type kubeObject interface { // getObject returns the first object of type T from the given map of kubernetes objects. // This helper can be used for doing post processing on the objects. func getObject[T kubeObject](manifests k8sutil.ObjectMap) T { + var ret T for _, obj := range manifests { if service, ok := obj.(T); ok { - return service + if ret != nil { + panic(fmt.Sprintf("found multiple objects of type %T", *new(T))) + } + ret = service } } - panic(fmt.Sprintf("could not find object of type %T", *new(T))) + if ret == nil { + panic(fmt.Sprintf("could not find object of type %T", *new(T))) + } + + return ret } // postProcessServiceMonitor updates the service monitor to work with the app-sre prometheus.