diff --git a/pkg/applicationprofilemanager/v1/applicationprofile_manager.go b/pkg/applicationprofilemanager/v1/applicationprofile_manager.go index 03d6b67c..36c28b9f 100644 --- a/pkg/applicationprofilemanager/v1/applicationprofile_manager.go +++ b/pkg/applicationprofilemanager/v1/applicationprofile_manager.go @@ -168,7 +168,7 @@ func (am *ApplicationProfileManager) monitorContainer(ctx context.Context, conta // adjust ticker after first tick if !watchedContainer.InitialDelayExpired { watchedContainer.InitialDelayExpired = true - watchedContainer.UpdateDataTicker.Reset(am.cfg.UpdateDataPeriod) + watchedContainer.UpdateDataTicker.Reset(utils.AddJitter(am.cfg.UpdateDataPeriod, am.cfg.MaxJitterPercentage)) } watchedContainer.SetStatus(utils.WatchedContainerStatusReady) am.saveProfile(ctx, watchedContainer, container.K8s.Namespace) @@ -531,7 +531,7 @@ func (am *ApplicationProfileManager) startApplicationProfiling(ctx context.Conte watchedContainer := &utils.WatchedContainerData{ ContainerID: container.Runtime.ContainerID, - UpdateDataTicker: time.NewTicker(utils.AddRandomDuration(5, 10, am.cfg.InitialDelay)), // get out of sync with the relevancy manager + UpdateDataTicker: time.NewTicker(utils.AddJitter(am.cfg.InitialDelay, am.cfg.MaxJitterPercentage)), SyncChannel: syncChannel, K8sContainerID: k8sContainerID, NsMntId: container.Mntns, diff --git a/pkg/config/config.go b/pkg/config/config.go index 14ea7752..8164a520 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,6 +17,7 @@ type Config struct { InitialDelay time.Duration `mapstructure:"initialDelay"` MaxSniffingTime time.Duration `mapstructure:"maxSniffingTimePerContainer"` UpdateDataPeriod time.Duration `mapstructure:"updateDataPeriod"` + MaxJitterPercentage int `mapstructure:"maxJitterPercentage"` EnableFullPathTracing bool `mapstructure:"fullPathTracingEnabled"` EnableApplicationProfile bool `mapstructure:"applicationProfileServiceEnabled"` EnableMalwareDetection bool `mapstructure:"malwareDetectionEnabled"` @@ -40,6 +41,7 @@ func LoadConfig(path string) (Config, error) { viper.SetDefault("fullPathTracingEnabled", true) viper.SetDefault("initialDelay", 2*time.Minute) viper.SetDefault("nodeProfileInterval", 10*time.Minute) + viper.SetDefault("maxJitterPercentage", 5) viper.AutomaticEnv() diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index a800cf09..e1faacaa 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -30,6 +30,7 @@ func TestLoadConfig(t *testing.T) { MaxSniffingTime: 6 * time.Hour, UpdateDataPeriod: 1 * time.Minute, NodeProfileInterval: 1 * time.Minute, + MaxJitterPercentage: 5, EnablePrometheusExporter: true, EnableRuntimeDetection: true, EnableSeccomp: true, diff --git a/pkg/containerwatcher/v1/container_watcher_private.go b/pkg/containerwatcher/v1/container_watcher_private.go index cd7bc080..da81b489 100644 --- a/pkg/containerwatcher/v1/container_watcher_private.go +++ b/pkg/containerwatcher/v1/container_watcher_private.go @@ -44,7 +44,7 @@ func (ch *IGContainerWatcher) containerCallback(notif containercollection.PubSub logger.L().Info("start monitor on container", helpers.String("container ID", notif.Container.Runtime.ContainerID), helpers.String("k8s workload", k8sContainerID)) // Check if Pod has a label of max sniffing time - sniffingTime := ch.cfg.MaxSniffingTime + sniffingTime := utils.AddJitter(ch.cfg.MaxSniffingTime, ch.cfg.MaxJitterPercentage) if podLabelMaxSniffingTime, ok := notif.Container.K8s.PodLabels[MaxSniffingTimeLabel]; ok { if duration, err := time.ParseDuration(podLabelMaxSniffingTime); err == nil { sniffingTime = duration diff --git a/pkg/malwaremanager/v1/malware_manager.go b/pkg/malwaremanager/v1/malware_manager.go index 1bb49ab8..e7a0ec1d 100644 --- a/pkg/malwaremanager/v1/malware_manager.go +++ b/pkg/malwaremanager/v1/malware_manager.go @@ -78,7 +78,7 @@ func (mm *MalwareManager) ContainerCallback(notif containercollection.PubSubEven return } - t := time.NewTicker(mm.cfg.InitialDelay) + t := time.NewTicker(utils.AddJitter(mm.cfg.InitialDelay, mm.cfg.MaxJitterPercentage)) switch notif.Type { case containercollection.EventTypeAddContainer: diff --git a/pkg/networkmanager/v2/network_manager.go b/pkg/networkmanager/v2/network_manager.go index 83258ce6..b56920d8 100644 --- a/pkg/networkmanager/v2/network_manager.go +++ b/pkg/networkmanager/v2/network_manager.go @@ -183,7 +183,7 @@ func (nm *NetworkManager) monitorContainer(ctx context.Context, container *conta // adjust ticker after first tick if !watchedContainer.InitialDelayExpired { watchedContainer.InitialDelayExpired = true - watchedContainer.UpdateDataTicker.Reset(nm.cfg.UpdateDataPeriod) + watchedContainer.UpdateDataTicker.Reset(utils.AddJitter(nm.cfg.UpdateDataPeriod, nm.cfg.MaxJitterPercentage)) } watchedContainer.SetStatus(utils.WatchedContainerStatusReady) nm.saveNetworkEvents(ctx, watchedContainer, container.K8s.Namespace) @@ -433,7 +433,7 @@ func (nm *NetworkManager) startNetworkMonitoring(ctx context.Context, container watchedContainer := &utils.WatchedContainerData{ ContainerID: container.Runtime.ContainerID, - UpdateDataTicker: time.NewTicker(utils.AddRandomDuration(5, 10, nm.cfg.InitialDelay)), // get out of sync with the relevancy manager + UpdateDataTicker: time.NewTicker(utils.AddJitter(nm.cfg.InitialDelay, nm.cfg.MaxJitterPercentage)), SyncChannel: syncChannel, K8sContainerID: k8sContainerID, NsMntId: container.Mntns, diff --git a/pkg/nodeprofilemanager/v1/nodeprofile_manager.go b/pkg/nodeprofilemanager/v1/nodeprofile_manager.go index f7104790..ce1985ed 100644 --- a/pkg/nodeprofilemanager/v1/nodeprofile_manager.go +++ b/pkg/nodeprofilemanager/v1/nodeprofile_manager.go @@ -51,7 +51,7 @@ var _ nodeprofilemanager.NodeProfileManagerClient = (*NodeProfileManager)(nil) func (n *NodeProfileManager) Start(ctx context.Context) { go func() { - time.Sleep(n.config.InitialDelay) + time.Sleep(utils.AddJitter(n.config.InitialDelay, n.config.MaxJitterPercentage)) for { time.Sleep(n.config.NodeProfileInterval) profile, err := n.getProfile() diff --git a/pkg/relevancymanager/v1/relevancy_manager.go b/pkg/relevancymanager/v1/relevancy_manager.go index ce350098..a7d708f8 100644 --- a/pkg/relevancymanager/v1/relevancy_manager.go +++ b/pkg/relevancymanager/v1/relevancy_manager.go @@ -253,7 +253,7 @@ func (rm *RelevancyManager) monitorContainer(ctx context.Context, container *con // adjust ticker after first tick if !watchedContainer.InitialDelayExpired { watchedContainer.InitialDelayExpired = true - watchedContainer.UpdateDataTicker.Reset(rm.cfg.UpdateDataPeriod) + watchedContainer.UpdateDataTicker.Reset(utils.AddJitter(rm.cfg.UpdateDataPeriod, rm.cfg.MaxJitterPercentage)) } // handle collection of relevant data rm.handleRelevancy(ctx, watchedContainer, container.Runtime.ContainerID) @@ -284,7 +284,7 @@ func (rm *RelevancyManager) startRelevancyProcess(ctx context.Context, container watchedContainer := &utils.WatchedContainerData{ ContainerID: container.Runtime.ContainerID, - UpdateDataTicker: time.NewTicker(rm.cfg.InitialDelay), + UpdateDataTicker: time.NewTicker(utils.AddJitter(rm.cfg.InitialDelay, rm.cfg.MaxJitterPercentage)), SyncChannel: make(chan error, 10), K8sContainerID: k8sContainerID, RelevantRelationshipsArtifactsByIdentifier: make(map[string]bool), diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index edd7b774..5fa71cac 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -165,11 +165,13 @@ func CreateK8sPodID(namespaceName string, podName string) string { return strings.Join([]string{namespaceName, podName}, "/") } -// AddRandomDuration adds between min and max seconds to duration -func AddRandomDuration(min, max int, duration time.Duration) time.Duration { - // we don't initialize the seed, so we will get the same sequence of random numbers every time - randomDuration := time.Duration(rand.Intn(max+1-min)+min) * time.Second - return randomDuration + duration +// AddJitter adds jitter percent to the duration +func AddJitter(duration time.Duration, maxJitterPercentage int) time.Duration { + if maxJitterPercentage == 0 { + return duration + } + jitter := 1 + rand.Intn(maxJitterPercentage)/100 + return duration * time.Duration(jitter) } func Atoi(s string) int { diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index cc44eed7..fb39940b 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -4,7 +4,6 @@ import ( "reflect" "strings" "testing" - "time" apitypes "github.com/armosec/armoapi-go/armotypes" "github.com/kubescape/k8s-interface/instanceidhandler/v1" @@ -145,42 +144,6 @@ func TestCreateK8sContainerID(t *testing.T) { } } -func TestRandomSleep(t *testing.T) { - type args struct { - min int - max int - } - tests := []struct { - name string - args args - }{ - { - name: "normal", - args: args{ - min: 1, - max: 3, - }, - }, - { - name: "min equals max", - args: args{ - min: 1, - max: 1, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - start := time.Now() - time.Sleep(AddRandomDuration(tt.args.min, tt.args.max, 0)) - elapsed := int(time.Since(start).Seconds()) - if elapsed < tt.args.min || elapsed > tt.args.max { - t.Errorf("AddRandomDuration() = %v, want between %v and %v", elapsed, tt.args.min, tt.args.max) - } - }) - } -} - func TestAtoi(t *testing.T) { type args struct { s string