Skip to content

Commit

Permalink
fix: Tolerate Pods without labels in metrics
Browse files Browse the repository at this point in the history
Some Pods may not have labels, for example the Pods created by our CLI
during the restore process.

The deployed sessions collector should not fail when it hits such a Pod.
Instead, it should just ignore it.
  • Loading branch information
MoritzWeber0 committed Jul 18, 2024
1 parent 92041c3 commit 9bc2bbb
Show file tree
Hide file tree
Showing 2 changed files with 54 additions and 36 deletions.
11 changes: 7 additions & 4 deletions backend/capellacollab/sessions/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,13 @@ def collect(self) -> t.Iterable[prometheus_client.core.Metric]:
)
operator = operators.get_operator()
pods = operator.get_pods(label_selector=None)
statuses = sorted(
(pod.metadata.labels.get("workload", ""), pod.status.phase.lower())
for pod in pods
)
statuses = []
for pod in pods:
labels = pod.metadata.labels
if labels is None or "workload" not in labels:
continue

statuses.append((labels["workload"], pod.status.phase.lower()))
for labels, g in itertools.groupby(statuses):
metric.add_metric(labels, len(list(g)))
yield metric
Expand Down
79 changes: 47 additions & 32 deletions backend/tests/test_metrics.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
# SPDX-License-Identifier: Apache-2.0

import pytest
from fastapi import testclient
from kubernetes import client as k8s_client

from capellacollab.sessions import metrics, operators
from capellacollab.sessions import metrics


def test_metrics_endpoint(client: testclient.TestClient):
Expand All @@ -13,7 +15,7 @@ def test_metrics_endpoint(client: testclient.TestClient):
assert "# HELP " in response.text


def test_database_sessions_metric(db):
def test_database_sessions_metric_empty():
collector = metrics.DatabaseSessionsCollector()

data = list(collector.collect())
Expand All @@ -22,10 +24,50 @@ def test_database_sessions_metric(db):
assert data[0].samples


def test_kubernetes_sessions_metric(monkeypatch):
monkeypatch.setattr(operators, "get_operator", MockOperator)
collector = metrics.DeployedSessionsCollector()
def test_kubernetes_sessions_metric(monkeypatch: pytest.MonkeyPatch):
def mock_list_namespaced_pod(
self, namespace: str, label_selector: str
) -> k8s_client.V1PodList:
return k8s_client.V1PodList(
items=[
k8s_client.V1Pod(
metadata=k8s_client.V1ObjectMeta(),
status=k8s_client.V1PodStatus(phase="running"),
),
k8s_client.V1Pod(
metadata=k8s_client.V1ObjectMeta(
labels={"workload": "job"}
),
status=k8s_client.V1PodStatus(phase="pending"),
),
k8s_client.V1Pod(
metadata=k8s_client.V1ObjectMeta(
labels={"workload": "job"}
),
status=k8s_client.V1PodStatus(phase="running"),
),
k8s_client.V1Pod(
metadata=k8s_client.V1ObjectMeta(
labels={"workload": "session"}
),
status=k8s_client.V1PodStatus(phase="running"),
),
k8s_client.V1Pod(
metadata=k8s_client.V1ObjectMeta(
labels={"workload": "session"}
),
status=k8s_client.V1PodStatus(phase="running"),
),
]
)

monkeypatch.setattr(
k8s_client.CoreV1Api,
"list_namespaced_pod",
mock_list_namespaced_pod,
)

collector = metrics.DeployedSessionsCollector()
data = list(collector.collect())

samples = {
Expand All @@ -38,30 +80,3 @@ def test_kubernetes_sessions_metric(monkeypatch):
("job", "pending"): 1,
("session", "running"): 2,
}


class MockOperator:
def get_pods(self, label_selector):
return [
attrdict(
metadata=attrdict(labels=dict(workload="job")),
status=attrdict(phase="running"),
),
attrdict(
metadata=attrdict(labels=dict(workload="job")),
status=attrdict(phase="pending"),
),
attrdict(
metadata=attrdict(labels=dict(workload="session")),
status=attrdict(phase="running"),
),
attrdict(
metadata=attrdict(labels=dict(workload="session")),
status=attrdict(phase="running"),
),
]


class attrdict(dict):
def __getattr__(self, name):
return self[name]

0 comments on commit 9bc2bbb

Please sign in to comment.