Skip to content

Commit

Permalink
Fix metrics-exporter test
Browse files Browse the repository at this point in the history
  • Loading branch information
@alex-zaitsev
alex-zaitsev committed Feb 12, 2024
1 parent 5a17901 commit 3236368
Show file tree
Hide file tree
Showing 2 changed files with 64 additions and 84 deletions.
10 changes: 6 additions & 4 deletions tests/e2e/manifests/chi/test-017-multi-version.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,17 @@ apiVersion: clickhouse.altinity.com/v1
kind: ClickHouseInstallation
metadata:
name: test-017-multi-version
labels:
clickhouse.altinity.com/chi: test-017-multi-version
spec:
templates:
podTemplates:
- name: v22.3
- name: v23.3
spec:
containers:
- name: clickhouse-pod
image: clickhouse/clickhouse-server:23.3
- name: v22.8
- name: v23.8
spec:
containers:
- name: clickhouse-pod
Expand All @@ -21,9 +23,9 @@ spec:
layout:
shards:
- templates:
podTemplate: v22.3
podTemplate: v23.3
- templates:
podTemplate: v22.8
podTemplate: v23.8
files:
users.d/remove_database_ordinary.xml: |
<yandex>
Expand Down
138 changes: 58 additions & 80 deletions tests/e2e/test_metrics_exporter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ def test_metrics_exporter_setup(self):


@TestScenario
@Name("Check metrics server state after reboot")
def test_metrics_exporter_reboot(self):
@Name("Test basic metrics exporter functionality")
def test_metrics_exporter_chi(self):
def check_monitoring_chi(operator_namespace, operator_pod, expect_result, max_retries=10):
with Then(f"metrics-exporter /chi endpoint result should return {expect_result}"):
for i in range(1, max_retries):
Expand All @@ -41,9 +41,30 @@ def check_monitoring_chi(operator_namespace, operator_pod, expect_result, max_re
out = json.loads(out)
if out == expect_result:
break
print(out)
print(expect_result)
with Then("Not ready. Wait for " + str(i * 5) + " seconds"):
time.sleep(i * 5)
assert out == expect_result, error()
def check_monitoring_metrics(operator_namespace, operator_pod, expect_result, max_retries=10):
with Then(f"metrics-exporter /metrics endpoint result should match with {expect_result}"):
found = 0
for i in range(1, max_retries):
url_cmd = util.make_http_get_request("127.0.0.1", "8888", "/metrics")
out = kubectl.launch(
f"exec {operator_pod} -c metrics-exporter -- {url_cmd}",
ns=operator_namespace,
)
found = 0
for string, exists in expect_result.items():
if exists == (string in out):
found = found + 1
print(f"FOUND: {string}")
if found == len(expect_result.items()):
break
with Then("Not ready. Wait for " + str(i * 5) + " seconds"):
time.sleep(i * 5)
assert found == len(expect_result.items()), error()

with Given("clickhouse-operator is installed"):
kubectl.wait_field(
Expand All @@ -60,29 +81,37 @@ def check_monitoring_chi(operator_namespace, operator_pod, expect_result, max_re
kubectl.delete_all_chi(self.context.test_namespace)
check_monitoring_chi(self.context.operator_namespace, operator_pod, [])
with And("created simple clickhouse installation"):
manifest = "../../docs/chi-examples/01-simple-layout-01-1shard-1repl.yaml"
manifest = "manifests/chi/test-017-multi-version.yaml"
kubectl.create_and_check(
manifest=manifest,
check={
"object_counts": {
"statefulset": 1,
"pod": 1,
"service": 2,
"statefulset": 2,
"pod": 2,
"service": 3,
},
"do_not_delete": True,
},
)
expected_chi = [
{
"namespace": "test",
"name": "simple-01",
"name": "test-017-multi-version",
"labels": {"clickhouse.altinity.com/chi": "test-017-multi-version"},
"annotations": {},
"clusters": [
{
"name": "simple",
"name": "default",
"hosts": [
{
"name": "0-0",
"hostname": "chi-simple-01-simple-0-0.test.svc.cluster.local",
"hostname": "chi-test-017-multi-version-default-0-0.test.svc.cluster.local",
"tcpPort": 9000,
"httpPort": 8123
},
{
"name": "1-0",
"hostname": "chi-test-017-multi-version-default-1-0.test.svc.cluster.local",
"tcpPort": 9000,
"httpPort": 8123
}
Expand All @@ -91,85 +120,35 @@ def check_monitoring_chi(operator_namespace, operator_pod, expect_result, max_re
]
}
]
with Then("Check both pods are monitored"):
check_monitoring_chi(self.context.operator_namespace, operator_pod, expected_chi)
with When("reboot metrics exporter"):
kubectl.launch(f"exec -n {self.context.operator_namespace} {operator_pod} -c metrics-exporter -- bash -c 'kill 1'")
time.sleep(15)
kubectl.wait_field(
"pods",
util.operator_label,
".status.containerStatuses[*].ready",
"true,true",
ns=self.context.operator_namespace,
)
with Then("check metrics exporter still contains chi objects"):
check_monitoring_chi(self.context.operator_namespace, operator_pod, expected_chi)
kubectl.delete(util.get_full_path(manifest, lookup_in_host=False), timeout=600)
check_monitoring_chi(self.context.operator_namespace, operator_pod, [])


@TestScenario
@Name("Check metrics server help with different clickhouse version")
def test_metrics_exporter_with_multiple_clickhouse_version(self):
def check_monitoring_metrics(operator_namespace, operator_pod, expect_result, max_retries=10):
with Then(f"metrics-exporter /metrics endpoint result should match with {expect_result}"):
for i in range(1, max_retries):
url_cmd = util.make_http_get_request("127.0.0.1", "8888", "/metrics")
out = kubectl.launch(
f"exec {operator_pod} -c metrics-exporter -- {url_cmd}",
ns=operator_namespace,
)
all_strings_expected_done = True
for string, exists in expect_result.items():
all_strings_expected_done = exists == (string in out)
if not all_strings_expected_done:
break

if all_strings_expected_done:
break
with Then("Not ready. Wait for " + str(i * 5) + " seconds"):
time.sleep(i * 5)
assert all_strings_expected_done, error()

with Given("clickhouse-operator pod exists"):
out = kubectl.launch("get pods -l app=clickhouse-operator", ns=self.context.operator_namespace).splitlines()[1]
operator_pod = re.split(r"[\t\r\n\s]+", out)[0]

with Then("check empty /metrics"):
kubectl.delete_all_chi(self.context.test_namespace)
check_monitoring_metrics(
self.context.operator_namespace,
operator_pod,
expect_result={
"chi_clickhouse_metric_VersionInteger": False,
},
)

with Then("Install multiple clickhouse version"):
manifest = "manifests/chi/test-017-multi-version.yaml"
kubectl.create_and_check(
manifest=manifest,
check={
"object_counts": {
"statefulset": 2,
"pod": 2,
"service": 3,
},
"do_not_delete": True,
},
)
with And("Check not empty /metrics"):
with Then("Check not empty /metrics"):
check_monitoring_metrics(
self.context.operator_namespace,
operator_pod,
expect_result={
"# HELP chi_clickhouse_metric_VersionInteger": True,
"# TYPE chi_clickhouse_metric_VersionInteger gauge": True,
'chi_clickhouse_metric_VersionInteger{chi="test-017-multi-version",hostname="chi-test-017-multi-version-default-0-0': True,
'chi_clickhouse_metric_VersionInteger{chi="test-017-multi-version",hostname="chi-test-017-multi-version-default-1-0': True,
'chi_clickhouse_metric_VersionInteger{chi="test-017-multi-version",clickhouse_altinity_com_chi="test-017-multi-version",hostname="chi-test-017-multi-version-default-0-0': True,
'chi_clickhouse_metric_VersionInteger{chi="test-017-multi-version",clickhouse_altinity_com_chi="test-017-multi-version",hostname="chi-test-017-multi-version-default-1-0': True,
},
)

with When("reboot metrics exporter"):
kubectl.launch(f"exec -n {self.context.operator_namespace} {operator_pod} -c metrics-exporter -- bash -c 'kill 1'")
time.sleep(15)
kubectl.wait_field(
"pods",
util.operator_label,
".status.containerStatuses[*].ready",
"true,true",
ns=self.context.operator_namespace,
)
with Then("check metrics exporter still contains chi objects"):
check_monitoring_chi(self.context.operator_namespace, operator_pod, expected_chi)
kubectl.delete(util.get_full_path(manifest, lookup_in_host=False), timeout=600)
check_monitoring_chi(self.context.operator_namespace, operator_pod, [])

with Then("check empty /metrics after delete namespace"):
kubectl.delete_all_chi(self.context.test_namespace)
check_monitoring_metrics(
Expand All @@ -196,8 +175,7 @@ def test(self):
util.install_operator_if_not_exist()
test_cases = [
test_metrics_exporter_setup,
test_metrics_exporter_reboot,
test_metrics_exporter_with_multiple_clickhouse_version,
test_metrics_exporter_chi,
]
for t in test_cases:
Scenario(test=t)()

0 comments on commit 3236368

Please sign in to comment.