feat: webhook metrics (#5602)

* feat: webhook metrics

* fix: remove resource

* docs: metrics

docs/docs/articles/metrics.md

@@ -21,6 +21,7 @@ The Testkube API Server exposes a `/metrics` endpoint that can be consumed by Pr
 * `testkube_testsuite_executions_duration_ms`- The duration of test suite executions.
 * `testkube_testworkflow_executions_duration_ms`- The duration of test workflow executions.
 * `testkube_testtrigger_event_count` - The total number of test trigger events.
+* `testkube_webhook_executions_count` - The total number of webhook executions.
 
 Note: as the metrics also include labels with the associated test name (see below), no metrics are produced unless some tests were run since last api-server restart 
 

internal/app/api/metrics/metrics.go

@@ -139,6 +139,11 @@ var testTriggerEventCount = promauto.NewCounterVec(prometheus.CounterOpts{
 	Help: "The total number of test trigger events",
 }, []string{"name", "resource", "eventType", "causes"})
 
+var webhookExecutionsCount = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "testkube_webhook_executions_count",
+	Help: "The total number of webhook executions",
+}, []string{"name", "eventType", "result"})
+
 func NewMetrics() Metrics {
 	return Metrics{
 		TestExecutionsCount:              testExecutionsCount,
@@ -166,6 +171,7 @@ func NewMetrics() Metrics {
 		TestWorkflowTemplateUpdates:      testWorkflowTemplateUpdatesCount,
 		TestWorkflowTemplateDeletes:      testWorkflowTemplateDeletesCount,
 		TestTriggerEventCount:            testTriggerEventCount,
+		WebhookEventCount:                webhookExecutionsCount,
 	}
 }
 
@@ -195,6 +201,7 @@ type Metrics struct {
 	TestWorkflowTemplateUpdates      *prometheus.CounterVec
 	TestWorkflowTemplateDeletes      *prometheus.CounterVec
 	TestTriggerEventCount            *prometheus.CounterVec
+	WebhookEventCount                *prometheus.CounterVec
 }
 
 func (m Metrics) IncAndObserveExecuteTest(execution testkube.Execution, dashboardURI string) {
@@ -498,3 +505,11 @@ func (m Metrics) IncTestTriggerEventCount(name, resource, eventType string, caus
 		"causes":    strings.Join(causes, ","),
 	}).Inc()
 }
+
+func (m Metrics) InWebhookEventCount(name, eventType, result string) {
+	m.WebhookEventCount.With(map[string]string{
+		"name":      name,
+		"eventType": eventType,
+		"result":    result,
+	}).Inc()
+}

internal/app/api/v1/server.go

@@ -169,7 +169,7 @@ func NewTestkubeAPI(
 	// will be reused in websockets handler
 	s.WebsocketLoader = ws.NewWebsocketLoader()
 
-	s.Events.Loader.Register(webhook.NewWebhookLoader(s.Log, webhookClient, templatesClient, testExecutionResults, testSuiteExecutionsResults, testWorkflowResults))
+	s.Events.Loader.Register(webhook.NewWebhookLoader(s.Log, webhookClient, templatesClient, testExecutionResults, testSuiteExecutionsResults, testWorkflowResults, metrics))
 	s.Events.Loader.Register(s.WebsocketLoader)
 	s.Events.Loader.Register(s.slackLoader)
 

pkg/event/kind/webhook/listener.go

@@ -12,6 +12,7 @@ import (
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 
+	v1 "github.com/kubeshop/testkube/internal/app/api/metrics"
 	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
 	"github.com/kubeshop/testkube/pkg/event/kind/common"
 	thttp "github.com/kubeshop/testkube/pkg/http"
@@ -30,7 +31,7 @@ func NewWebhookListener(name, uri, selector string, events []testkube.EventType,
 	onStateChange bool,
 	testExecutionResults result.Repository,
 	testSuiteExecutionResults testresult.Repository,
-	testWorkflowExecutionResults testworkflow.Repository) *WebhookListener {
+	testWorkflowExecutionResults testworkflow.Repository, metrics v1.Metrics) *WebhookListener {
 	return &WebhookListener{
 		name:                         name,
 		Uri:                          uri,
@@ -46,6 +47,7 @@ func NewWebhookListener(name, uri, selector string, events []testkube.EventType,
 		testExecutionResults:         testExecutionResults,
 		testSuiteExecutionResults:    testSuiteExecutionResults,
 		testWorkflowExecutionResults: testWorkflowExecutionResults,
+		metrics:                      metrics,
 	}
 }
 
@@ -64,6 +66,7 @@ type WebhookListener struct {
 	testExecutionResults         result.Repository
 	testSuiteExecutionResults    testresult.Repository
 	testWorkflowExecutionResults testworkflow.Repository
+	metrics                      v1.Metrics
 }
 
 func (l *WebhookListener) Name() string {
@@ -108,19 +111,37 @@ func (l *WebhookListener) Disabled() bool {
 }
 
 func (l *WebhookListener) Notify(event testkube.Event) (result testkube.EventResult) {
+	defer func() {
+		var eventType, res string
+		if event.Type_ != nil {
+			eventType = string(*event.Type_)
+		}
+
+		res = "success"
+		if result.Error() != "" {
+			res = "error"
+		}
+
+		l.metrics.InWebhookEventCount(l.name, eventType, res)
+	}()
+
 	switch {
 	case l.disabled:
 		l.Log.With(event.Log()...).Debug("webhook listener is disabled")
-		return testkube.NewSuccessEventResult(event.Id, "webhook listener is disabled")
+		result = testkube.NewSuccessEventResult(event.Id, "webhook listener is disabled")
+		return
 	case event.TestExecution != nil && event.TestExecution.DisableWebhooks:
 		l.Log.With(event.Log()...).Debug("webhook listener is disabled for test execution")
-		return testkube.NewSuccessEventResult(event.Id, "webhook listener is disabled for test execution")
+		result = testkube.NewSuccessEventResult(event.Id, "webhook listener is disabled for test execution")
+		return
 	case event.TestSuiteExecution != nil && event.TestSuiteExecution.DisableWebhooks:
 		l.Log.With(event.Log()...).Debug("webhook listener is disabled for test suite execution")
-		return testkube.NewSuccessEventResult(event.Id, "webhook listener is disabled for test suite execution")
+		result = testkube.NewSuccessEventResult(event.Id, "webhook listener is disabled for test suite execution")
+		return
 	case event.TestWorkflowExecution != nil && event.TestWorkflowExecution.DisableWebhooks:
 		l.Log.With(event.Log()...).Debug("webhook listener is disabled for test workflow execution")
-		return testkube.NewSuccessEventResult(event.Id, "webhook listener is disabled for test workflow execution")
+		result = testkube.NewSuccessEventResult(event.Id, "webhook listener is disabled for test workflow execution")
+		return
 	}
 
 	if l.onStateChange {
@@ -141,7 +162,8 @@ func (l *WebhookListener) Notify(event testkube.Event) (result testkube.EventRes
 		var data []byte
 		data, err = l.processTemplate("payload", l.payloadTemplate, event)
 		if err != nil {
-			return testkube.NewFailedEventResult(event.Id, err)
+			result = testkube.NewFailedEventResult(event.Id, err)
+			return
 		}
 
 		_, err = body.Write(data)
@@ -157,18 +179,21 @@ func (l *WebhookListener) Notify(event testkube.Event) (result testkube.EventRes
 	if err != nil {
 		err = errors.Wrap(err, "webhook send encode error")
 		log.Errorw("webhook send encode error", "error", err)
-		return testkube.NewFailedEventResult(event.Id, err)
+		result = testkube.NewFailedEventResult(event.Id, err)
+		return
 	}
 
 	data, err := l.processTemplate("uri", l.Uri, event)
 	if err != nil {
-		return testkube.NewFailedEventResult(event.Id, err)
+		result = testkube.NewFailedEventResult(event.Id, err)
+		return
 	}
 
 	request, err := http.NewRequest(http.MethodPost, string(data), body)
 	if err != nil {
 		log.Errorw("webhook request creating error", "error", err)
-		return testkube.NewFailedEventResult(event.Id, err)
+		result = testkube.NewFailedEventResult(event.Id, err)
+		return
 	}
 
 	request.Header.Set("Content-Type", "application/json")
@@ -177,7 +202,8 @@ func (l *WebhookListener) Notify(event testkube.Event) (result testkube.EventRes
 		for i := range values {
 			data, err = l.processTemplate("header", *values[i], event)
 			if err != nil {
-				return testkube.NewFailedEventResult(event.Id, err)
+				result = testkube.NewFailedEventResult(event.Id, err)
+				return
 			}
 
 			*values[i] = string(data)
@@ -189,26 +215,30 @@ func (l *WebhookListener) Notify(event testkube.Event) (result testkube.EventRes
 	resp, err := l.HttpClient.Do(request)
 	if err != nil {
 		log.Errorw("webhook send error", "error", err)
-		return testkube.NewFailedEventResult(event.Id, err)
+		result = testkube.NewFailedEventResult(event.Id, err)
+		return
 	}
 	defer resp.Body.Close()
 
 	data, err = io.ReadAll(resp.Body)
 	if err != nil {
 		log.Errorw("webhook read response error", "error", err)
-		return testkube.NewFailedEventResult(event.Id, err)
+		result = testkube.NewFailedEventResult(event.Id, err)
+		return
 	}
 
 	responseStr := string(data)
 
 	if resp.StatusCode >= 400 {
 		err := fmt.Errorf("webhook response with bad status code: %d", resp.StatusCode)
 		log.Errorw("webhook send error", "error", err, "status", resp.StatusCode)
-		return testkube.NewFailedEventResult(event.Id, err).WithResult(responseStr)
+		result = testkube.NewFailedEventResult(event.Id, err).WithResult(responseStr)
+		return
 	}
 
 	log.Debugw("got webhook send result", "response", responseStr)
-	return testkube.NewSuccessEventResult(event.Id, responseStr)
+	result = testkube.NewSuccessEventResult(event.Id, responseStr)
+	return
 }
 
 func (l *WebhookListener) Kind() string {

pkg/event/kind/webhook/listener_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
+	v1 "github.com/kubeshop/testkube/internal/app/api/metrics"
 	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
 )
 
@@ -33,7 +34,7 @@ func TestWebhookListener_Notify(t *testing.T) {
 		svr := httptest.NewServer(testHandler)
 		defer svr.Close()
 
-		l := NewWebhookListener("l1", svr.URL, "", testEventTypes, "", "", nil, false, false, nil, nil, nil)
+		l := NewWebhookListener("l1", svr.URL, "", testEventTypes, "", "", nil, false, false, nil, nil, nil, v1.NewMetrics())
 
 		// when
 		r := l.Notify(testkube.Event{
@@ -55,7 +56,7 @@ func TestWebhookListener_Notify(t *testing.T) {
 		svr := httptest.NewServer(testHandler)
 		defer svr.Close()
 
-		l := NewWebhookListener("l1", svr.URL, "", testEventTypes, "", "", nil, false, false, nil, nil, nil)
+		l := NewWebhookListener("l1", svr.URL, "", testEventTypes, "", "", nil, false, false, nil, nil, nil, v1.NewMetrics())
 
 		// when
 		r := l.Notify(testkube.Event{
@@ -72,7 +73,7 @@ func TestWebhookListener_Notify(t *testing.T) {
 		t.Parallel()
 		// given
 
-		s := NewWebhookListener("l1", "http://baduri.badbadbad", "", testEventTypes, "", "", nil, false, false, nil, nil, nil)
+		s := NewWebhookListener("l1", "http://baduri.badbadbad", "", testEventTypes, "", "", nil, false, false, nil, nil, nil, v1.NewMetrics())
 
 		// when
 		r := s.Notify(testkube.Event{
@@ -105,7 +106,7 @@ func TestWebhookListener_Notify(t *testing.T) {
 		svr := httptest.NewServer(testHandler)
 		defer svr.Close()
 
-		l := NewWebhookListener("l1", svr.URL, "", testEventTypes, "field", "", nil, false, false, nil, nil, nil)
+		l := NewWebhookListener("l1", svr.URL, "", testEventTypes, "field", "", nil, false, false, nil, nil, nil, v1.NewMetrics())
 
 		// when
 		r := l.Notify(testkube.Event{
@@ -132,7 +133,7 @@ func TestWebhookListener_Notify(t *testing.T) {
 		defer svr.Close()
 
 		l := NewWebhookListener("l1", svr.URL, "", testEventTypes, "", "{\"id\": \"{{ .Id }}\"}",
-			map[string]string{"Content-Type": "application/json"}, false, false, nil, nil, nil)
+			map[string]string{"Content-Type": "application/json"}, false, false, nil, nil, nil, v1.NewMetrics())
 
 		// when
 		r := l.Notify(testkube.Event{
@@ -149,7 +150,7 @@ func TestWebhookListener_Notify(t *testing.T) {
 		t.Parallel()
 		// given
 
-		s := NewWebhookListener("l1", "http://baduri.badbadbad", "", testEventTypes, "", "", nil, true, false, nil, nil, nil)
+		s := NewWebhookListener("l1", "http://baduri.badbadbad", "", testEventTypes, "", "", nil, true, false, nil, nil, nil, v1.NewMetrics())
 
 		// when
 		r := s.Notify(testkube.Event{

pkg/event/kind/webhook/loader.go

@@ -7,6 +7,7 @@ import (
 
 	executorsv1 "github.com/kubeshop/testkube-operator/api/executor/v1"
 	templatesclientv1 "github.com/kubeshop/testkube-operator/pkg/client/templates/v1"
+	v1 "github.com/kubeshop/testkube/internal/app/api/metrics"
 	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
 	"github.com/kubeshop/testkube/pkg/event/kind/common"
 	"github.com/kubeshop/testkube/pkg/mapper/webhooks"
@@ -24,6 +25,7 @@ type WebhooksLister interface {
 
 func NewWebhookLoader(log *zap.SugaredLogger, webhooksClient WebhooksLister, templatesClient templatesclientv1.Interface,
 	testExecutionResults result.Repository, testSuiteExecutionResults testresult.Repository, testWorkflowExecutionResults testworkflow.Repository,
+	metrics v1.Metrics,
 ) *WebhooksLoader {
 	return &WebhooksLoader{
 		log:                          log,
@@ -32,6 +34,7 @@ func NewWebhookLoader(log *zap.SugaredLogger, webhooksClient WebhooksLister, tem
 		testExecutionResults:         testExecutionResults,
 		testSuiteExecutionResults:    testSuiteExecutionResults,
 		testWorkflowExecutionResults: testWorkflowExecutionResults,
+		metrics:                      metrics,
 	}
 }
 
@@ -42,6 +45,7 @@ type WebhooksLoader struct {
 	testExecutionResults         result.Repository
 	testSuiteExecutionResults    testresult.Repository
 	testWorkflowExecutionResults testworkflow.Repository
+	metrics                      v1.Metrics
 }
 
 func (r WebhooksLoader) Kind() string {
@@ -79,7 +83,7 @@ func (r WebhooksLoader) Load() (listeners common.Listeners, err error) {
 		name := fmt.Sprintf("%s.%s", webhook.ObjectMeta.Namespace, webhook.ObjectMeta.Name)
 		listeners = append(listeners, NewWebhookListener(name, webhook.Spec.Uri, webhook.Spec.Selector, types,
 			webhook.Spec.PayloadObjectField, payloadTemplate, webhook.Spec.Headers, webhook.Spec.Disabled,
-			webhook.Spec.OnStateChange, r.testExecutionResults, r.testSuiteExecutionResults, r.testWorkflowExecutionResults))
+			webhook.Spec.OnStateChange, r.testExecutionResults, r.testSuiteExecutionResults, r.testWorkflowExecutionResults, r.metrics))
 	}
 
 	return listeners, nil

pkg/event/kind/webhook/loader_test.go

@@ -9,6 +9,7 @@ import (
 
 	executorsv1 "github.com/kubeshop/testkube-operator/api/executor/v1"
 	templatesclientv1 "github.com/kubeshop/testkube-operator/pkg/client/templates/v1"
+	v1 "github.com/kubeshop/testkube/internal/app/api/metrics"
 )
 
 type DummyLoader struct {
@@ -29,7 +30,7 @@ func TestWebhookLoader(t *testing.T) {
 	defer mockCtrl.Finish()
 
 	mockTemplatesClient := templatesclientv1.NewMockInterface(mockCtrl)
-	webhooksLoader := NewWebhookLoader(zap.NewNop().Sugar(), &DummyLoader{}, mockTemplatesClient, nil, nil, nil)
+	webhooksLoader := NewWebhookLoader(zap.NewNop().Sugar(), &DummyLoader{}, mockTemplatesClient, nil, nil, nil, v1.NewMetrics())
 	listeners, err := webhooksLoader.Load()
 
 	assert.Equal(t, 1, len(listeners))