Merge pull request #359 from kubescape/jitter

add jitter to all sniffing tickers
kubescape · Sep 2, 2024 · 33f53e1 · 33f53e1
2 parents 1fc3466 + 8ea67fb
commit 33f53e1
Show file tree

Hide file tree

Showing 10 changed files with 19 additions and 51 deletions.
diff --git a/pkg/applicationprofilemanager/v1/applicationprofile_manager.go b/pkg/applicationprofilemanager/v1/applicationprofile_manager.go
@@ -168,7 +168,7 @@ func (am *ApplicationProfileManager) monitorContainer(ctx context.Context, conta
 			// adjust ticker after first tick
 			if !watchedContainer.InitialDelayExpired {
 				watchedContainer.InitialDelayExpired = true
-				watchedContainer.UpdateDataTicker.Reset(am.cfg.UpdateDataPeriod)
+				watchedContainer.UpdateDataTicker.Reset(utils.AddJitter(am.cfg.UpdateDataPeriod, am.cfg.MaxJitterPercentage))
 			}
 			watchedContainer.SetStatus(utils.WatchedContainerStatusReady)
 			am.saveProfile(ctx, watchedContainer, container.K8s.Namespace)
@@ -531,7 +531,7 @@ func (am *ApplicationProfileManager) startApplicationProfiling(ctx context.Conte
 
 	watchedContainer := &utils.WatchedContainerData{
 		ContainerID:      container.Runtime.ContainerID,
-		UpdateDataTicker: time.NewTicker(utils.AddRandomDuration(5, 10, am.cfg.InitialDelay)), // get out of sync with the relevancy manager
+		UpdateDataTicker: time.NewTicker(utils.AddJitter(am.cfg.InitialDelay, am.cfg.MaxJitterPercentage)),
 		SyncChannel:      syncChannel,
 		K8sContainerID:   k8sContainerID,
 		NsMntId:          container.Mntns,

diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -17,6 +17,7 @@ type Config struct {
 	InitialDelay             time.Duration             `mapstructure:"initialDelay"`
 	MaxSniffingTime          time.Duration             `mapstructure:"maxSniffingTimePerContainer"`
 	UpdateDataPeriod         time.Duration             `mapstructure:"updateDataPeriod"`
+	MaxJitterPercentage      int                       `mapstructure:"maxJitterPercentage"`
 	EnableFullPathTracing    bool                      `mapstructure:"fullPathTracingEnabled"`
 	EnableApplicationProfile bool                      `mapstructure:"applicationProfileServiceEnabled"`
 	EnableMalwareDetection   bool                      `mapstructure:"malwareDetectionEnabled"`
@@ -40,6 +41,7 @@ func LoadConfig(path string) (Config, error) {
 	viper.SetDefault("fullPathTracingEnabled", true)
 	viper.SetDefault("initialDelay", 2*time.Minute)
 	viper.SetDefault("nodeProfileInterval", 10*time.Minute)
+	viper.SetDefault("maxJitterPercentage", 5)
 
 	viper.AutomaticEnv()
 

diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go
@@ -30,6 +30,7 @@ func TestLoadConfig(t *testing.T) {
 				MaxSniffingTime:          6 * time.Hour,
 				UpdateDataPeriod:         1 * time.Minute,
 				NodeProfileInterval:      1 * time.Minute,
+				MaxJitterPercentage:      5,
 				EnablePrometheusExporter: true,
 				EnableRuntimeDetection:   true,
 				EnableSeccomp:            true,

diff --git a/pkg/containerwatcher/v1/container_watcher_private.go b/pkg/containerwatcher/v1/container_watcher_private.go
@@ -44,7 +44,7 @@ func (ch *IGContainerWatcher) containerCallback(notif containercollection.PubSub
 		logger.L().Info("start monitor on container", helpers.String("container ID", notif.Container.Runtime.ContainerID), helpers.String("k8s workload", k8sContainerID))
 
 		// Check if Pod has a label of max sniffing time
-		sniffingTime := ch.cfg.MaxSniffingTime
+		sniffingTime := utils.AddJitter(ch.cfg.MaxSniffingTime, ch.cfg.MaxJitterPercentage)
 		if podLabelMaxSniffingTime, ok := notif.Container.K8s.PodLabels[MaxSniffingTimeLabel]; ok {
 			if duration, err := time.ParseDuration(podLabelMaxSniffingTime); err == nil {
 				sniffingTime = duration

diff --git a/pkg/malwaremanager/v1/malware_manager.go b/pkg/malwaremanager/v1/malware_manager.go
@@ -78,7 +78,7 @@ func (mm *MalwareManager) ContainerCallback(notif containercollection.PubSubEven
 		return
 	}
 
-	t := time.NewTicker(mm.cfg.InitialDelay)
+	t := time.NewTicker(utils.AddJitter(mm.cfg.InitialDelay, mm.cfg.MaxJitterPercentage))
 
 	switch notif.Type {
 	case containercollection.EventTypeAddContainer:

diff --git a/pkg/networkmanager/v2/network_manager.go b/pkg/networkmanager/v2/network_manager.go
@@ -183,7 +183,7 @@ func (nm *NetworkManager) monitorContainer(ctx context.Context, container *conta
 			// adjust ticker after first tick
 			if !watchedContainer.InitialDelayExpired {
 				watchedContainer.InitialDelayExpired = true
-				watchedContainer.UpdateDataTicker.Reset(nm.cfg.UpdateDataPeriod)
+				watchedContainer.UpdateDataTicker.Reset(utils.AddJitter(nm.cfg.UpdateDataPeriod, nm.cfg.MaxJitterPercentage))
 			}
 			watchedContainer.SetStatus(utils.WatchedContainerStatusReady)
 			nm.saveNetworkEvents(ctx, watchedContainer, container.K8s.Namespace)
@@ -433,7 +433,7 @@ func (nm *NetworkManager) startNetworkMonitoring(ctx context.Context, container
 
 	watchedContainer := &utils.WatchedContainerData{
 		ContainerID:      container.Runtime.ContainerID,
-		UpdateDataTicker: time.NewTicker(utils.AddRandomDuration(5, 10, nm.cfg.InitialDelay)), // get out of sync with the relevancy manager
+		UpdateDataTicker: time.NewTicker(utils.AddJitter(nm.cfg.InitialDelay, nm.cfg.MaxJitterPercentage)),
 		SyncChannel:      syncChannel,
 		K8sContainerID:   k8sContainerID,
 		NsMntId:          container.Mntns,

diff --git a/pkg/nodeprofilemanager/v1/nodeprofile_manager.go b/pkg/nodeprofilemanager/v1/nodeprofile_manager.go
@@ -51,7 +51,7 @@ var _ nodeprofilemanager.NodeProfileManagerClient = (*NodeProfileManager)(nil)
 
 func (n *NodeProfileManager) Start(ctx context.Context) {
 	go func() {
-		time.Sleep(n.config.InitialDelay)
+		time.Sleep(utils.AddJitter(n.config.InitialDelay, n.config.MaxJitterPercentage))
 		for {
 			time.Sleep(n.config.NodeProfileInterval)
 			profile, err := n.getProfile()

diff --git a/pkg/relevancymanager/v1/relevancy_manager.go b/pkg/relevancymanager/v1/relevancy_manager.go
@@ -253,7 +253,7 @@ func (rm *RelevancyManager) monitorContainer(ctx context.Context, container *con
 			// adjust ticker after first tick
 			if !watchedContainer.InitialDelayExpired {
 				watchedContainer.InitialDelayExpired = true
-				watchedContainer.UpdateDataTicker.Reset(rm.cfg.UpdateDataPeriod)
+				watchedContainer.UpdateDataTicker.Reset(utils.AddJitter(rm.cfg.UpdateDataPeriod, rm.cfg.MaxJitterPercentage))
 			}
 			// handle collection of relevant data
 			rm.handleRelevancy(ctx, watchedContainer, container.Runtime.ContainerID)
@@ -284,7 +284,7 @@ func (rm *RelevancyManager) startRelevancyProcess(ctx context.Context, container
 
 	watchedContainer := &utils.WatchedContainerData{
 		ContainerID:      container.Runtime.ContainerID,
-		UpdateDataTicker: time.NewTicker(rm.cfg.InitialDelay),
+		UpdateDataTicker: time.NewTicker(utils.AddJitter(rm.cfg.InitialDelay, rm.cfg.MaxJitterPercentage)),
 		SyncChannel:      make(chan error, 10),
 		K8sContainerID:   k8sContainerID,
 		RelevantRelationshipsArtifactsByIdentifier: make(map[string]bool),

diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go
@@ -165,11 +165,13 @@ func CreateK8sPodID(namespaceName string, podName string) string {
 	return strings.Join([]string{namespaceName, podName}, "/")
 }
 
-// AddRandomDuration adds between min and max seconds to duration
-func AddRandomDuration(min, max int, duration time.Duration) time.Duration {
-	// we don't initialize the seed, so we will get the same sequence of random numbers every time
-	randomDuration := time.Duration(rand.Intn(max+1-min)+min) * time.Second
-	return randomDuration + duration
+// AddJitter adds jitter percent to the duration
+func AddJitter(duration time.Duration, maxJitterPercentage int) time.Duration {
+	if maxJitterPercentage == 0 {
+		return duration
+	}
+	jitter := 1 + rand.Intn(maxJitterPercentage)/100
+	return duration * time.Duration(jitter)
 }
 
 func Atoi(s string) int {

diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go
@@ -4,7 +4,6 @@ import (
 	"reflect"
 	"strings"
 	"testing"
-	"time"
 
 	apitypes "github.com/armosec/armoapi-go/armotypes"
 	"github.com/kubescape/k8s-interface/instanceidhandler/v1"
@@ -145,42 +144,6 @@ func TestCreateK8sContainerID(t *testing.T) {
 	}
 }
 
-func TestRandomSleep(t *testing.T) {
-	type args struct {
-		min int
-		max int
-	}
-	tests := []struct {
-		name string
-		args args
-	}{
-		{
-			name: "normal",
-			args: args{
-				min: 1,
-				max: 3,
-			},
-		},
-		{
-			name: "min equals max",
-			args: args{
-				min: 1,
-				max: 1,
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			start := time.Now()
-			time.Sleep(AddRandomDuration(tt.args.min, tt.args.max, 0))
-			elapsed := int(time.Since(start).Seconds())
-			if elapsed < tt.args.min || elapsed > tt.args.max {
-				t.Errorf("AddRandomDuration() = %v, want between %v and %v", elapsed, tt.args.min, tt.args.max)
-			}
-		})
-	}
-}
-
 func TestAtoi(t *testing.T) {
 	type args struct {
 		s string