diff --git a/Makefile b/Makefile
index c11380c509d..58444c7dda9 100644
--- a/Makefile
+++ b/Makefile
@@ -99,7 +99,7 @@ IMAGE_BUILD_FLAGS ?= --build-arg USE_LOCAL=false
 PROMETHEUS_CONFIG_YAML = ./config/monitoring/prometheus/apps/prometheus-configs.yaml
 PROMETHEUS_CONFIG_DIR = ./config/monitoring/prometheus/apps
 PROMETHEUS_TEST_DIR = ./tests/prometheus_unit_tests
-PROMETHEUS_ALERT_TESTS = $(shell find $(PROMETHEUS_TEST_DIR) -name "*.unit-tests.yaml")
+PROMETHEUS_ALERT_TESTS = $(wildcard $(PROMETHEUS_TEST_DIR)/*.unit-tests.yaml)
 
 ALERT_SEVERITY = critical
 
@@ -387,15 +387,18 @@ CLEANFILES += cover.out
 $(PROMETHEUS_TEST_DIR)/%.rules.yaml: $(PROMETHEUS_TEST_DIR)/%.unit-tests.yaml $(PROMETHEUS_CONFIG_YAML) $(YQ)
 	$(YQ) eval ".data.\"$(@F:.rules.yaml=.rules)\"" $(PROMETHEUS_CONFIG_YAML) > $@
 
+PROMETHEUS_ALERT_RULES := $(PROMETHEUS_ALERT_TESTS:.unit-tests.yaml=.rules.yaml)
+
 # Run prometheus-alert-unit-tests
 .PHONY: test-alerts
-test-alerts: $(PROMETHEUS_ALERT_TESTS:.unit-tests.yaml=.rules.yaml)
+test-alerts: $(PROMETHEUS_ALERT_RULES)
 	promtool test rules $(PROMETHEUS_ALERT_TESTS)
 
 #Check for alerts without unit-tests
 .PHONY: check-prometheus-alert-unit-tests
-check-prometheus-alert-unit-tests: $(PROMETHEUS_ALERT_TESTS:.unit-tests.yaml=.rules.yaml)
+check-prometheus-alert-unit-tests: $(PROMETHEUS_ALERT_RULES)
 	./tests/prometheus_unit_tests/scripts/check_alert_tests.sh $(PROMETHEUS_CONFIG_YAML) $(PROMETHEUS_TEST_DIR) $(ALERT_SEVERITY)
+CLEANFILES += $(PROMETHEUS_ALERT_RULES)
 
 .PHONY: e2e-test
 e2e-test: ## Run e2e tests for the controller
@@ -406,4 +409,3 @@ clean: $(GOLANGCI_LINT)
 	$(GOLANGCI_LINT) cache clean
 	chmod u+w -R $(LOCALBIN) # envtest makes its dir RO
 	rm -rf $(CLEANFILES)
-	rm $(PROMETHEUS_ALERT_TESTS:.unit-tests.yaml=.rules.yaml)