From 6b38cea84cfcefb28157d6f2fc958e6942fe3a4a Mon Sep 17 00:00:00 2001
From: Karol Kokoszka <karol.kokoszka@scylladb.com>
Date: Tue, 26 Mar 2024 11:53:20 +0100
Subject: [PATCH] feat(scheduler): add error logging on run interruption

---
 pkg/scheduler/listener.go         | 34 +++++++++++++++++++++++++++----
 pkg/scheduler/scheduler.go        |  4 ++--
 pkg/scheduler/scheduler_test.go   |  4 ++--
 pkg/service/scheduler/listener.go |  2 +-
 4 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/pkg/scheduler/listener.go b/pkg/scheduler/listener.go
index 7ef5cd5ae8..c7f2078338 100644
--- a/pkg/scheduler/listener.go
+++ b/pkg/scheduler/listener.go
@@ -5,6 +5,8 @@ package scheduler
 import (
 	"context"
 	"time"
+
+	"github.com/scylladb/go-log"
 )
 
 // Listener specifies pluggable hooks for scheduler events.
@@ -14,8 +16,8 @@ type Listener[K comparable] interface {
 	OnSchedulerStop(context.Context)
 	OnRunStart(ctx *RunContext[K])
 	OnRunSuccess(ctx *RunContext[K])
-	OnRunStop(ctx *RunContext[K])
-	OnRunWindowEnd(ctx *RunContext[K])
+	OnRunStop(ctx *RunContext[K], err error)
+	OnRunWindowEnd(ctx *RunContext[K], err error)
 	OnRunError(ctx *RunContext[K], err error)
 	OnSchedule(ctx context.Context, key K, begin, end time.Time, retno int8)
 	OnUnschedule(ctx context.Context, key K)
@@ -40,10 +42,10 @@ func (l nopListener[K]) OnRunStart(*RunContext[K]) {
 func (l nopListener[K]) OnRunSuccess(*RunContext[K]) {
 }
 
-func (l nopListener[K]) OnRunStop(*RunContext[K]) {
+func (l nopListener[K]) OnRunStop(*RunContext[K], error) {
 }
 
-func (l nopListener[K]) OnRunWindowEnd(*RunContext[K]) {
+func (l nopListener[K]) OnRunWindowEnd(*RunContext[K], error) {
 }
 
 func (l nopListener[K]) OnRunError(*RunContext[K], error) {
@@ -74,3 +76,27 @@ func (l nopListener[K]) OnSleep(context.Context, K, time.Duration) {
 func NopListener[K comparable]() Listener[K] {
 	return nopListener[K]{}
 }
+
+type errorLogListener[K comparable] struct {
+	nopListener[K]
+	logger log.Logger
+}
+
+func (l errorLogListener[K]) OnRunError(ctx *RunContext[K], err error) {
+	l.logger.Error(ctx, "OnRunError", "key", ctx.Key, "retry", ctx.Retry, "error", err)
+}
+
+func (l errorLogListener[K]) OnRunWindowEnd(ctx *RunContext[K], err error) {
+	l.logger.Info(ctx, "OnRunWindowEnd", "key", ctx.Key, "retry", ctx.Retry, "error", err)
+}
+
+func (l errorLogListener[K]) OnRunStop(ctx *RunContext[K], err error) {
+	l.logger.Info(ctx, "OnRunStop", "key", ctx.Key, "retry", ctx.Retry, "error", err)
+}
+
+// ErrorLogListener returns listener that logs errors.
+func ErrorLogListener[K comparable](logger log.Logger) Listener[K] {
+	return errorLogListener[K]{
+		logger: logger,
+	}
+}
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
index b0916ea1ae..65de2887be 100644
--- a/pkg/scheduler/scheduler.go
+++ b/pkg/scheduler/scheduler.go
@@ -413,9 +413,9 @@ func (s *Scheduler[K]) onRunEnd(ctx *RunContext[K]) {
 	case err == nil:
 		s.listener.OnRunSuccess(ctx)
 	case errors.Is(err, context.Canceled):
-		s.listener.OnRunStop(ctx)
+		s.listener.OnRunStop(ctx, err)
 	case errors.Is(err, context.DeadlineExceeded):
-		s.listener.OnRunWindowEnd(ctx)
+		s.listener.OnRunWindowEnd(ctx, err)
 	default:
 		s.listener.OnRunError(ctx, err)
 	}
diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go
index ef1e22eeb5..a19c22f090 100644
--- a/pkg/scheduler/scheduler_test.go
+++ b/pkg/scheduler/scheduler_test.go
@@ -196,11 +196,11 @@ func (l logListener) OnRunSuccess(ctx *testRunContext) {
 	l.logger.Info(ctx, "OnRunSuccess", "key", ctx.Key, "retry", ctx.Retry)
 }
 
-func (l logListener) OnRunStop(ctx *testRunContext) {
+func (l logListener) OnRunStop(ctx *testRunContext, err error) {
 	l.logger.Info(ctx, "OnRunStop", "key", ctx.Key, "retry", ctx.Retry)
 }
 
-func (l logListener) OnRunWindowEnd(ctx *testRunContext) {
+func (l logListener) OnRunWindowEnd(ctx *testRunContext, err error) {
 	l.logger.Info(ctx, "OnRunWindowEnd", "key", ctx.Key, "retry", ctx.Retry)
 }
 
diff --git a/pkg/service/scheduler/listener.go b/pkg/service/scheduler/listener.go
index 8d42f10cc5..4565808ae2 100644
--- a/pkg/service/scheduler/listener.go
+++ b/pkg/service/scheduler/listener.go
@@ -21,7 +21,7 @@ type schedulerListener struct {
 
 func newSchedulerListener(find func(key Key) (taskInfo, bool), logger log.Logger) schedulerListener {
 	return schedulerListener{
-		Listener: scheduler.NopListener[Key](),
+		Listener: scheduler.ErrorLogListener[Key](logger),
 		find:     find,
 		logger:   logger,
 	}