Skip to content

Commit

Permalink
add external labels to receives (#642)
Browse files Browse the repository at this point in the history 
Signed-off-by: Thibault Mange <[email protected]>
  • Loading branch information
@thibaultmg
thibaultmg authored Nov 8, 2023
1 parent 8c6b412 commit a18ecf9
Show file tree
Hide file tree
Showing 9 changed files with 55 additions and 14 deletions.
7 changes: 5 additions & 2 deletions ...p-sre-stage-01/rhobs/default/observatorium-metrics-receive-ingestor-default-template.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,7 @@ objects:
- receive
- --grpc-address=0.0.0.0:10901
- --http-address=0.0.0.0:10902
- --label=receive=true
- --label=receive-replica=$(NAME)
- --label=replica="$(POD_NAME)"
- --log.format=logfmt
- --log.level=${LOG_LEVEL}
- --objstore.config=$(OBJSTORE_CONFIG)
Expand Down Expand Up @@ -190,6 +189,10 @@ objects:
bucket: $(OBJ_STORE_BUCKET)
endpoint: $(OBJ_STORE_ENDPOINT)
region: $(OBJ_STORE_REGION)
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
image: quay.io/thanos/thanos:v0.32.4
imagePullPolicy: IfNotPresent
livenessProbe:
Expand Down
1 change: 1 addition & 0 deletions resources/services/app-sre-stage-01/rhobs/observatorium-metrics-receive-router-template.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,7 @@ objects:
- receive
- --grpc-address=0.0.0.0:10901
- --http-address=0.0.0.0:10902
- --label=receive="true"
- --log.format=logfmt
- --log.level=${LOG_LEVEL}
- --receive.hashrings-algorithm=ketama
Expand Down
7 changes: 5 additions & 2 deletions ...ces/app-sre-stage-01/rhobs/rhel/observatorium-metrics-receive-ingestor-rhel-template.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,7 @@ objects:
- receive
- --grpc-address=0.0.0.0:10901
- --http-address=0.0.0.0:10902
- --label=receive=true
- --label=receive-replica=$(NAME)
- --label=replica="$(POD_NAME)"
- --log.format=logfmt
- --log.level=${LOG_LEVEL}
- --objstore.config=$(OBJSTORE_CONFIG)
Expand Down Expand Up @@ -190,6 +189,10 @@ objects:
bucket: $(OBJ_STORE_BUCKET)
endpoint: $(OBJ_STORE_ENDPOINT)
region: $(OBJ_STORE_REGION)
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
image: quay.io/thanos/thanos:v0.32.4
imagePullPolicy: IfNotPresent
livenessProbe:
Expand Down
7 changes: 5 additions & 2 deletions ...e-stage-01/rhobs/telemeter/observatorium-metrics-receive-ingestor-telemeter-template.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,7 @@ objects:
- receive
- --grpc-address=0.0.0.0:10901
- --http-address=0.0.0.0:10902
- --label=receive=true
- --label=receive-replica=$(NAME)
- --label=replica="$(POD_NAME)"
- --log.format=logfmt
- --log.level=${LOG_LEVEL}
- --objstore.config=$(OBJSTORE_CONFIG)
Expand Down Expand Up @@ -190,6 +189,10 @@ objects:
bucket: $(OBJ_STORE_BUCKET)
endpoint: $(OBJ_STORE_ENDPOINT)
region: $(OBJ_STORE_REGION)
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
image: quay.io/thanos/thanos:v0.32.4
imagePullPolicy: IfNotPresent
livenessProbe:
Expand Down
7 changes: 5 additions & 2 deletions ...emeter-prod-01/rhobs/default/observatorium-metrics-receive-ingestor-default-template.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,7 @@ objects:
- receive
- --grpc-address=0.0.0.0:10901
- --http-address=0.0.0.0:10902
- --label=receive=true
- --label=receive-replica=$(NAME)
- --label=replica="$(POD_NAME)"
- --log.format=logfmt
- --log.level=${LOG_LEVEL}
- --objstore.config=$(OBJSTORE_CONFIG)
Expand Down Expand Up @@ -190,6 +189,10 @@ objects:
bucket: $(OBJ_STORE_BUCKET)
endpoint: $(OBJ_STORE_ENDPOINT)
region: $(OBJ_STORE_REGION)
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
image: quay.io/thanos/thanos:v0.32.4
imagePullPolicy: IfNotPresent
livenessProbe:
Expand Down
1 change: 1 addition & 0 deletions ...urces/services/telemeter-prod-01/rhobs/observatorium-metrics-receive-router-template.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,7 @@ objects:
- receive
- --grpc-address=0.0.0.0:10901
- --http-address=0.0.0.0:10902
- --label=receive="true"
- --log.format=logfmt
- --log.level=${LOG_LEVEL}
- --receive.hashrings-algorithm=ketama
Expand Down
7 changes: 5 additions & 2 deletions ...es/telemeter-prod-01/rhobs/rhel/observatorium-metrics-receive-ingestor-rhel-template.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,7 @@ objects:
- receive
- --grpc-address=0.0.0.0:10901
- --http-address=0.0.0.0:10902
- --label=receive=true
- --label=receive-replica=$(NAME)
- --label=replica="$(POD_NAME)"
- --log.format=logfmt
- --log.level=${LOG_LEVEL}
- --objstore.config=$(OBJSTORE_CONFIG)
Expand Down Expand Up @@ -190,6 +189,10 @@ objects:
bucket: $(OBJ_STORE_BUCKET)
endpoint: $(OBJ_STORE_ENDPOINT)
region: $(OBJ_STORE_REGION)
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
image: quay.io/thanos/thanos:v0.32.4
imagePullPolicy: IfNotPresent
livenessProbe:
Expand Down
7 changes: 5 additions & 2 deletions ...er-prod-01/rhobs/telemeter/observatorium-metrics-receive-ingestor-telemeter-template.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,7 @@ objects:
- receive
- --grpc-address=0.0.0.0:10901
- --http-address=0.0.0.0:10902
- --label=receive=true
- --label=receive-replica=$(NAME)
- --label=replica="$(POD_NAME)"
- --log.format=logfmt
- --log.level=${LOG_LEVEL}
- --objstore.config=$(OBJSTORE_CONFIG)
Expand Down Expand Up @@ -190,6 +189,10 @@ objects:
bucket: $(OBJ_STORE_BUCKET)
endpoint: $(OBJ_STORE_ENDPOINT)
region: $(OBJ_STORE_REGION)
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
image: quay.io/thanos/thanos:v0.32.4
imagePullPolicy: IfNotPresent
livenessProbe:
Expand Down
25 changes: 23 additions & 2 deletions services_go/observatorium/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,12 @@ func (o ObservatoriumMetrics) makeReceiveRouter() encoding.Encoder {
ServiceName: "thanos-receive-router",
},
}
router.Options.Label = []receive.Label{
{
Key: "receive",
Value: "\"true\"",
},
}

receiveLimits := receive.NewReceiveLimitsConfig()
receiveLimits.WriteLimits.DefaultLimits = o.ReceiveLimitsDefault
Expand Down Expand Up @@ -253,6 +259,7 @@ func (o ObservatoriumMetrics) makeTenantReceiveIngestor(instanceCfg *Observatori
ingestor.PodResources.Limits[corev1.ResourceMemory] = resource.MustParse("24Gi")
ingestor.Env = deleteObjStoreEnv(ingestor.Env) // delete the default objstore env vars
ingestor.Env = append(ingestor.Env, objStoreEnvVars(instanceCfg.ObjStoreSecret)...)
ingestor.Env = append(ingestor.Env, k8sutil.NewEnvFromField("POD_NAME", "metadata.name"))
ingestor.Sidecars = []k8sutil.ContainerProvider{makeJaegerAgent("observatorium-tools")}

// Router config
Expand All @@ -266,6 +273,12 @@ func (o ObservatoriumMetrics) makeTenantReceiveIngestor(instanceCfg *Observatori
ServiceName: "thanos-receive-router",
},
}
ingestor.Options.Label = []receive.Label{
{
Key: "replica",
Value: "\"$(POD_NAME)\"",
},
}

// Execute preManifestsHook
if instanceCfg.ReceiveIngestorPreManifestsHook != nil {
Expand Down Expand Up @@ -757,13 +770,21 @@ type kubeObject interface {
// getObject returns the first object of type T from the given map of kubernetes objects.
// This helper can be used for doing post processing on the objects.
func getObject[T kubeObject](manifests k8sutil.ObjectMap) T {
var ret T
for _, obj := range manifests {
if service, ok := obj.(T); ok {
return service
if ret != nil {
panic(fmt.Sprintf("found multiple objects of type %T", *new(T)))
}
ret = service
}
}

panic(fmt.Sprintf("could not find object of type %T", *new(T)))
if ret == nil {
panic(fmt.Sprintf("could not find object of type %T", *new(T)))
}

return ret
}

// postProcessServiceMonitor updates the service monitor to work with the app-sre prometheus.
Expand Down

0 comments on commit a18ecf9

Please sign in to comment.