Skip to content

Commit

Permalink
Change extract-alerts to associative array + githubCI fix
Browse files Browse the repository at this point in the history
  • Loading branch information
biswassri committed Nov 14, 2024
1 parent e4502b6 commit ab09b9c
Show file tree
Hide file tree
Showing 4 changed files with 35 additions and 40 deletions.
10 changes: 8 additions & 2 deletions .github/workflows/prometheus-unit-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,16 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v4
with:
go-version-file: go.mod

- name: Install Promtool
run: |
apt-get update
apt-get install -y prometheus
go install github.com/prometheus/prometheus/cmd/promtool@latest
export PATH=$PATH:$(go env GOPATH)/bin
promtool --version
- name: Run prometheus-unit-tests
run : make test-alerts
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ IMAGE_BUILD_FLAGS ?= --build-arg USE_LOCAL=false
PROMETHEUS_CONFIG_YAML = ./config/monitoring/prometheus/apps/prometheus-configs.yaml
PROMETHEUS_CONFIG_DIR = ./config/monitoring/prometheus/apps/
GENERATED_ALERT_DIR = ./tests/prometheus_unit_tests/
CRITICAL_SEVERITY="critical"
ALERT_SEVERITY = critical

# Read any custom variables overrides from a local.mk file. This will only be read if it exists in the
# same directory as this Makefile. Variables can be specified in the standard format supported by
Expand Down Expand Up @@ -392,7 +392,7 @@ test-alerts: extract-alert-rules
#Check for alerts without unit-tests
.PHONY: check-prometheus-alert-unit-tests
check-prometheus-alert-unit-tests: extract-alert-rules
./tests/prometheus_unit_tests/scripts/check_alert_tests.sh $(PROMETHEUS_CONFIG_YAML) $(GENERATED_ALERT_DIR)
./tests/prometheus_unit_tests/scripts/check_alert_tests.sh $(PROMETHEUS_CONFIG_YAML) $(GENERATED_ALERT_DIR) $(ALERT_SEVERITY)

.PHONY: e2e-test
e2e-test: ## Run e2e tests for the controller
Expand Down
3 changes: 2 additions & 1 deletion tests/prometheus_unit_tests/scripts/check_alert_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,15 @@

PROMETHEUS_CONFIG_YAML=$1
UNIT_TEST_DIR=$2
ALERT_SEVERITY=$3

# Collect all alerts from the configuration file
while IFS= read -r ALERT; do
ALL_ALERTS+=("$ALERT")
done < <(yq -N e '.data[]
| from_yaml
| .groups[].rules[]
| select(.alert != "DeadManSnitch" and .labels.severity == "critical")
| select(.alert != "DeadManSnitch" and .labels.severity == strenv(ALERT_SEVERITY))
| .alert' "${PROMETHEUS_CONFIG_YAML}")

# Collect all alerts from the unit test files
Expand Down
58 changes: 23 additions & 35 deletions tests/prometheus_unit_tests/scripts/extract_alerts.sh
Original file line number Diff line number Diff line change
@@ -1,42 +1,30 @@
#!/bin/bash
#!/usr/bin/env bash
set -e

INPUT_YAML="$1"
OUTPUT_DIR="$2"

mkdir -p "$OUTPUT_DIR"

# Define RULES_FILES and output files
RULES_FILES=(
"rhods-dashboard-alerting.rules"
"model-mesh-alerting.rules"
"trustyai-alerting.rules"
"odh-model-controller-alerting.rules"
"workbenches-alerting.rules"
"data-science-pipelines-operator-alerting.rules"
"kserve-alerting.rules"
"kueue-alerting.rules"
"ray-alerting.rules"
"codeflare-alerting.rules"
"trainingoperator-alerting.rules"
)

ALERT_FILES=(
"dashboard_alerts.yaml"
"model_mesh_alerts.yaml"
"trustyai_alerts.yaml"
"model_controller_alerts.yaml"
"workbenches_alerts.yaml"
"data_science_pipelines_operator_alerts.yaml"
"kserve_alerts.yaml"
"kueue_alerts.yaml"
"kuberay_alerts.yaml"
"codeflare_alerts.yaml"
"training_operator_alerts.yaml"
# Define an associative array mapping rule files to alert files
declare -A RULES_TO_ALERTS=(
[rhods-dashboard-alerting.rules]="dashboard_alerts.yaml"
[model-mesh-alerting.rules]="model_mesh_alerts.yaml"
[trustyai-alerting.rules]="trustyai_alerts.yaml"
[odh-model-controller-alerting.rules]="model_controller_alerts.yaml"
[workbenches-alerting.rules]="workbenches_alerts.yaml"
[data-science-pipelines-operator-alerting.rules]="data_science_pipelines_operator_alerts.yaml"
[kserve-alerting.rules]="kserve_alerts.yaml"
[kueue-alerting.rules]="kueue_alerts.yaml"
[ray-alerting.rules]="kuberay_alerts.yaml"
[codeflare-alerting.rules]="codeflare_alerts.yaml"
[trainingoperator-alerting.rules]="training_operator_alerts.yaml"
)

for i in "${!RULES_FILES[@]}"; do
RULE_FILE="${RULES_FILES[$i]}"
ALERT_FILE="${ALERT_FILES[$i]}"
echo "Extracting $RULE_FILE to $OUTPUT_DIR/$ALERT_FILE"
yq ".data.\"$RULE_FILE\"" "$INPUT_YAML" > "$OUTPUT_DIR/$ALERT_FILE"
for RULE_FILE in "${!RULES_TO_ALERTS[@]}"; do
ALERT_FILE="${RULES_TO_ALERTS[$RULE_FILE]}"

echo "key: $RULE_FILE"
echo "value: $ALERT_FILE"

echo "Extracting $RULE_FILE to $OUTPUT_DIR/$ALERT_FILE"
yq ".data.\"$RULE_FILE\"" "$INPUT_YAML" > "$OUTPUT_DIR/$ALERT_FILE"
done

0 comments on commit ab09b9c

Please sign in to comment.