✨ Expose CronJob schedula via config (#938)

It is now possible to change the CronJob schedule for the different types of CronJobs via the `MondooAuditConfig`. Fixes #888 Signed-off-by: Christian Zunker <[email protected]>
mondoohq · Nov 16, 2023 · bf8c252 · bf8c252
1 parent 52435d6
commit bf8c252
Show file tree

Hide file tree

Showing 16 changed files with 248 additions and 15 deletions.
diff --git a/api/v1alpha2/mondooauditconfig_types.go b/api/v1alpha2/mondooauditconfig_types.go
@@ -86,11 +86,15 @@ type KubernetesResources struct {
 	// DEPRECATED: ContainerImageScanning determines whether container images are being scanned. The current implementation
 	// runs a separate job once every 24h that scans the container images running in the cluster.
 	ContainerImageScanning bool `json:"containerImageScanning,omitempty"`
+	// Specify a custom crontab schedule for the Kubernetes resource scanning job. If not specified, the default schedule is used.
+	Schedule string `json:"schedule,omitempty"`
 }
 
 type Nodes struct {
 	Enable    bool                        `json:"enable,omitempty"`
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
+	// Specify a custom crontab schedule for the node scanning job. If not specified, the default schedule is used.
+	Schedule string `json:"schedule,omitempty"`
 }
 
 type Admission struct {
@@ -118,6 +122,8 @@ type Admission struct {
 type Containers struct {
 	Enable    bool                        `json:"enable,omitempty"`
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
+	// Specify a custom crontab schedule for the container image scanning job. If not specified, the default schedule is used.
+	Schedule string `json:"schedule,omitempty"`
 }
 
 type Image struct {

diff --git a/config/crd/bases/k8s.mondoo.com_mondooauditconfigs.yaml b/config/crd/bases/k8s.mondoo.com_mondooauditconfigs.yaml
@@ -143,6 +143,11 @@ spec:
                           Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
                         type: object
                     type: object
+                  schedule:
+                    description: Specify a custom crontab schedule for the container
+                      image scanning job. If not specified, the default schedule is
+                      used.
+                    type: string
                 type: object
               filtering:
                 properties:
@@ -175,6 +180,11 @@ spec:
                     type: boolean
                   enable:
                     type: boolean
+                  schedule:
+                    description: Specify a custom crontab schedule for the Kubernetes
+                      resource scanning job. If not specified, the default schedule
+                      is used.
+                    type: string
                 type: object
               mondooCredsSecretRef:
                 description: Config is an example field of MondooAuditConfig. Edit
@@ -252,6 +262,10 @@ spec:
                           Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
                         type: object
                     type: object
+                  schedule:
+                    description: Specify a custom crontab schedule for the node scanning
+                      job. If not specified, the default schedule is used.
+                    type: string
                 type: object
               scanner:
                 description: Scanner defines the settings for the Mondoo scanner that

diff --git a/controllers/container_image/deployment_handler.go b/controllers/container_image/deployment_handler.go
@@ -125,6 +125,7 @@ func (n *DeploymentHandler) syncCronJob(ctx context.Context) error {
 		logger.Info("Created CronJob", "namespace", desired.Namespace, "name", desired.Name)
 	} else if !k8s.AreCronJobsEqual(*existing, *desired) {
 		existing.Spec.JobTemplate = desired.Spec.JobTemplate
+		existing.Spec.Schedule = desired.Spec.Schedule
 		existing.SetOwnerReferences(desired.GetOwnerReferences())
 
 		if err := n.KubeClient.Update(ctx, existing); err != nil {

diff --git a/controllers/container_image/deployment_handler_test.go b/controllers/container_image/deployment_handler_test.go
@@ -59,9 +59,6 @@ func (s *DeploymentHandlerSuite) TestReconcile_Create() {
 	s.NoError(err)
 	s.True(result.IsZero())
 
-	nodes := &corev1.NodeList{}
-	s.NoError(d.KubeClient.List(s.ctx, nodes))
-
 	image, err := s.containerImageResolver.CnspecImage("", "", false)
 	s.NoError(err)
 
@@ -92,9 +89,6 @@ func (s *DeploymentHandlerSuite) TestReconcile_CreateWithCustomImage() {
 	s.NoError(err)
 	s.True(result.IsZero())
 
-	nodes := &corev1.NodeList{}
-	s.NoError(d.KubeClient.List(s.ctx, nodes))
-
 	image, err := s.containerImageResolver.CnspecImage("ubuntu", "22.04", false)
 	s.NoError(err)
 
@@ -115,6 +109,51 @@ func (s *DeploymentHandlerSuite) TestReconcile_CreateWithCustomImage() {
 	s.Equal(expected, created)
 }
 
+func (s *DeploymentHandlerSuite) TestReconcile_CreateWithCustomSchedule() {
+	d := s.createDeploymentHandler()
+
+	customSchedule := "0 0 * * *"
+	s.auditConfig.Spec.Containers.Schedule = customSchedule
+
+	result, err := d.Reconcile(s.ctx)
+	s.NoError(err)
+	s.True(result.IsZero())
+
+	image, err := s.containerImageResolver.CnspecImage("", "", false)
+	s.NoError(err)
+
+	expected := CronJob(image, "", test.KubeSystemNamespaceUid, "", s.auditConfig, mondoov1alpha2.MondooOperatorConfig{})
+
+	created := &batchv1.CronJob{}
+	created.Name = expected.Name
+	created.Namespace = expected.Namespace
+	s.NoError(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(created), created))
+
+	s.Equal(created.Spec.Schedule, customSchedule)
+}
+
+func (s *DeploymentHandlerSuite) TestReconcile_CreateWithCustomScheduleFail() {
+	d := s.createDeploymentHandler()
+
+	customSchedule := "this is not valid"
+	s.auditConfig.Spec.Containers.Schedule = customSchedule
+
+	result, err := d.Reconcile(s.ctx)
+	s.NoError(err)
+	s.True(result.IsZero())
+
+	image, err := s.containerImageResolver.CnspecImage("", "", false)
+	s.NoError(err)
+
+	expected := CronJob(image, "", test.KubeSystemNamespaceUid, "", s.auditConfig, mondoov1alpha2.MondooOperatorConfig{})
+	created := &batchv1.CronJob{}
+	created.Name = expected.Name
+	created.Namespace = expected.Namespace
+	s.NoError(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(created), created))
+
+	s.NotEqual(created.Spec.Schedule, customSchedule)
+}
+
 func (s *DeploymentHandlerSuite) TestReconcile_Create_PrivateRegistriesSecret() {
 	d := s.createDeploymentHandler()
 
@@ -135,9 +174,6 @@ func (s *DeploymentHandlerSuite) TestReconcile_Create_PrivateRegistriesSecret()
 	s.NoError(err)
 	s.True(result.IsZero())
 
-	nodes := &corev1.NodeList{}
-	s.NoError(d.KubeClient.List(s.ctx, nodes))
-
 	image, err := s.containerImageResolver.CnspecImage("", "", false)
 	s.NoError(err)
 
@@ -177,9 +213,6 @@ func (s *DeploymentHandlerSuite) TestReconcile_Create_ConsoleIntegration() {
 	s.NoError(err)
 	s.True(result.IsZero())
 
-	nodes := &corev1.NodeList{}
-	s.NoError(d.KubeClient.List(s.ctx, nodes))
-
 	image, err := s.containerImageResolver.CnspecImage("", "", false)
 	s.NoError(err)
 

diff --git a/controllers/container_image/resources.go b/controllers/container_image/resources.go
@@ -8,6 +8,8 @@ import (
 	"strings"
 	"time"
 
+	// That's the mod k8s relies on https://github.com/kubernetes/kubernetes/blob/master/go.mod#L63
+	"github.com/robfig/cron/v3"
 	"go.mondoo.com/cnquery/v9/providers-sdk/v1/inventory"
 	"go.mondoo.com/mondoo-operator/api/v1alpha2"
 	"go.mondoo.com/mondoo-operator/pkg/constants"
@@ -49,6 +51,15 @@ func CronJob(image, integrationMrn, clusterUid, privateImageScanningSecretName s
 	// We want to start the cron job one minute after it was enabled.
 	cronStart := time.Now().Add(1 * time.Minute)
 	cronTab := fmt.Sprintf("%d %d * * *", cronStart.Minute(), cronStart.Hour())
+	if m.Spec.Containers.Schedule != "" {
+		_, err := cron.ParseStandard(m.Spec.Containers.Schedule)
+		if err != nil {
+			logger.Error(err, "invalid cron schedule specified in MondooAuditConfig Spec.Containers.Schedule; using default")
+		} else {
+			logger.Info("using cron custom schedule", "crontab", m.Spec.Containers.Schedule)
+			cronTab = m.Spec.Containers.Schedule
+		}
+	}
 
 	cronjob := &batchv1.CronJob{
 		ObjectMeta: metav1.ObjectMeta{

diff --git a/controllers/k8s_scan/deployment_handler.go b/controllers/k8s_scan/deployment_handler.go
@@ -93,6 +93,7 @@ func (n *DeploymentHandler) syncCronJob(ctx context.Context) error {
 		logger.Info("Created CronJob", "namespace", desired.Namespace, "name", desired.Name)
 	} else if !k8s.AreCronJobsEqual(*existing, *desired) {
 		existing.Spec.JobTemplate = desired.Spec.JobTemplate
+		existing.Spec.Schedule = desired.Spec.Schedule
 		existing.SetOwnerReferences(desired.GetOwnerReferences())
 
 		if err := n.KubeClient.Update(ctx, existing); err != nil {

diff --git a/controllers/k8s_scan/deployment_handler_test.go b/controllers/k8s_scan/deployment_handler_test.go
@@ -319,6 +319,64 @@ func (s *DeploymentHandlerSuite) TestReconcile_Disable() {
 	s.Equal(0, len(cronJobs.Items))
 }
 
+func (s *DeploymentHandlerSuite) TestReconcile_CreateWithCustomSchedule() {
+	d := s.createDeploymentHandler()
+
+	scanApiUrl := scanapi.ScanApiServiceUrl(*d.Mondoo)
+	s.scanApiStoreMock.EXPECT().Add(&scan_api_store.ScanApiStoreAddOpts{
+		Url:   scanApiUrl,
+		Token: "token",
+	}).Times(1)
+
+	customSchedule := "0 0 * * *"
+	s.auditConfig.Spec.KubernetesResources.Schedule = customSchedule
+
+	result, err := d.Reconcile(s.ctx)
+	s.NoError(err)
+	s.True(result.IsZero())
+
+	image, err := s.containerImageResolver.CnspecImage("", "", false)
+	s.NoError(err)
+
+	expected := CronJob(image, "", test.KubeSystemNamespaceUid, s.auditConfig)
+
+	created := &batchv1.CronJob{}
+	created.Name = expected.Name
+	created.Namespace = expected.Namespace
+	s.NoError(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(created), created))
+
+	s.Equal(created.Spec.Schedule, customSchedule)
+}
+
+func (s *DeploymentHandlerSuite) TestReconcile_CreateWithCustomScheduleFail() {
+	d := s.createDeploymentHandler()
+
+	scanApiUrl := scanapi.ScanApiServiceUrl(*d.Mondoo)
+	s.scanApiStoreMock.EXPECT().Add(&scan_api_store.ScanApiStoreAddOpts{
+		Url:   scanApiUrl,
+		Token: "token",
+	}).Times(1)
+
+	customSchedule := "this is not valid"
+	s.auditConfig.Spec.KubernetesResources.Schedule = customSchedule
+
+	result, err := d.Reconcile(s.ctx)
+	s.NoError(err)
+	s.True(result.IsZero())
+
+	image, err := s.containerImageResolver.CnspecImage("", "", false)
+	s.NoError(err)
+
+	expected := CronJob(image, "", test.KubeSystemNamespaceUid, s.auditConfig)
+
+	created := &batchv1.CronJob{}
+	created.Name = expected.Name
+	created.Namespace = expected.Namespace
+	s.NoError(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(created), created))
+
+	s.NotEqual(created.Spec.Schedule, customSchedule)
+}
+
 func (s *DeploymentHandlerSuite) createDeploymentHandler() DeploymentHandler {
 	return DeploymentHandler{
 		KubeClient:             s.fakeClientBuilder.Build(),

diff --git a/controllers/k8s_scan/resources.go b/controllers/k8s_scan/resources.go
@@ -8,6 +8,8 @@ import (
 	"strings"
 	"time"
 
+	// That's the mod k8s relies on https://github.com/kubernetes/kubernetes/blob/master/go.mod#L63
+	"github.com/robfig/cron/v3"
 	"go.mondoo.com/mondoo-operator/api/v1alpha2"
 	"go.mondoo.com/mondoo-operator/controllers/scanapi"
 	"go.mondoo.com/mondoo-operator/pkg/feature_flags"
@@ -24,6 +26,15 @@ func CronJob(image, integrationMrn, clusterUid string, m v1alpha2.MondooAuditCon
 	ls := CronJobLabels(m)
 
 	cronTab := fmt.Sprintf("%d * * * *", time.Now().Add(1*time.Minute).Minute())
+	if m.Spec.KubernetesResources.Schedule != "" {
+		_, err := cron.ParseStandard(m.Spec.KubernetesResources.Schedule)
+		if err != nil {
+			logger.Error(err, "invalid cron schedule specified in MondooAuditConfig Spec.KubernetesResources.Schedule; using default")
+		} else {
+			logger.Info("using cron custom schedule", "crontab", m.Spec.KubernetesResources.Schedule)
+			cronTab = m.Spec.KubernetesResources.Schedule
+		}
+	}
 	scanApiUrl := scanapi.ScanApiServiceUrl(m)
 
 	containerArgs := []string{

diff --git a/controllers/nodes/deployment_handler.go b/controllers/nodes/deployment_handler.go
@@ -103,6 +103,7 @@ func (n *DeploymentHandler) syncCronJob(ctx context.Context) error {
 
 		if !k8s.AreCronJobsEqual(*existing, *desired) {
 			existing.Spec.JobTemplate = desired.Spec.JobTemplate
+			existing.Spec.Schedule = desired.Spec.Schedule
 			existing.SetOwnerReferences(desired.GetOwnerReferences())
 
 			if err := n.KubeClient.Update(ctx, existing); err != nil {

diff --git a/controllers/nodes/deployment_handler_test.go b/controllers/nodes/deployment_handler_test.go
@@ -457,6 +457,60 @@ func (s *DeploymentHandlerSuite) TestReconcile_DisableNodeScanning() {
 	s.Equal(0, len(cronJobs.Items))
 }
 
+func (s *DeploymentHandlerSuite) TestReconcile_CreateWithCustomSchedule() {
+	s.seedNodes()
+	d := s.createDeploymentHandler()
+
+	customSchedule := "0 0 * * *"
+	s.auditConfig.Spec.Nodes.Schedule = customSchedule
+
+	result, err := d.Reconcile(s.ctx)
+	s.NoError(err)
+	s.True(result.IsZero())
+
+	nodes := &corev1.NodeList{}
+	s.NoError(d.KubeClient.List(s.ctx, nodes))
+
+	image, err := s.containerImageResolver.CnspecImage("", "", false)
+	s.NoError(err)
+
+	expected := CronJob(image, nodes.Items[0], s.auditConfig, false, v1alpha2.MondooOperatorConfig{})
+
+	created := &batchv1.CronJob{}
+	created.Name = expected.Name
+	created.Namespace = expected.Namespace
+	s.NoError(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(created), created))
+
+	s.Equal(created.Spec.Schedule, customSchedule)
+}
+
+func (s *DeploymentHandlerSuite) TestReconcile_CreateWithCustomScheduleFail() {
+	s.seedNodes()
+	d := s.createDeploymentHandler()
+
+	customSchedule := "this is not valid"
+	s.auditConfig.Spec.Nodes.Schedule = customSchedule
+
+	result, err := d.Reconcile(s.ctx)
+	s.NoError(err)
+	s.True(result.IsZero())
+
+	nodes := &corev1.NodeList{}
+	s.NoError(d.KubeClient.List(s.ctx, nodes))
+
+	image, err := s.containerImageResolver.CnspecImage("", "", false)
+	s.NoError(err)
+
+	expected := CronJob(image, nodes.Items[0], s.auditConfig, false, v1alpha2.MondooOperatorConfig{})
+
+	created := &batchv1.CronJob{}
+	created.Name = expected.Name
+	created.Namespace = expected.Namespace
+	s.NoError(d.KubeClient.Get(s.ctx, client.ObjectKeyFromObject(created), created))
+
+	s.NotEqual(created.Spec.Schedule, customSchedule)
+}
+
 func (s *DeploymentHandlerSuite) createDeploymentHandler() DeploymentHandler {
 	return DeploymentHandler{
 		KubeClient:             s.fakeClientBuilder.Build(),

diff --git a/controllers/nodes/resources.go b/controllers/nodes/resources.go
@@ -16,6 +16,8 @@ import (
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/yaml"
 
+	// That's the mod k8s relies on https://github.com/kubernetes/kubernetes/blob/master/go.mod#L63
+	"github.com/robfig/cron/v3"
 	"go.mondoo.com/cnquery/v9/providers-sdk/v1/inventory"
 	"go.mondoo.com/mondoo-operator/api/v1alpha2"
 	"go.mondoo.com/mondoo-operator/controllers/scanapi"
@@ -38,6 +40,15 @@ func CronJob(image string, node corev1.Node, m v1alpha2.MondooAuditConfig, isOpe
 	ls := CronJobLabels(m)
 
 	cronTab := fmt.Sprintf("%d * * * *", time.Now().Add(1*time.Minute).Minute())
+	if m.Spec.Nodes.Schedule != "" {
+		_, err := cron.ParseStandard(m.Spec.Nodes.Schedule)
+		if err != nil {
+			logger.Error(err, "invalid cron schedule specified in MondooAuditConfig Spec.Nodes.Schedule; using default")
+		} else {
+			logger.Info("using cron custom schedule", "crontab", m.Spec.Nodes.Schedule)
+			cronTab = m.Spec.Nodes.Schedule
+		}
+	}
 	unsetHostPath := corev1.HostPathUnset
 
 	name := "cnspec"
@@ -193,7 +204,7 @@ func CronJob(image string, node corev1.Node, m v1alpha2.MondooAuditConfig, isOpe
 func GarbageCollectCronJob(image, clusterUid string, m v1alpha2.MondooAuditConfig) *batchv1.CronJob {
 	ls := CronJobLabels(m)
 
-	cronTab := fmt.Sprintf("%d */2 * * *", time.Now().Add(1*time.Minute).Minute())
+	cronTab := fmt.Sprintf("%d */12 * * *", time.Now().Add(1*time.Minute).Minute())
 	scanApiUrl := scanapi.ScanApiServiceUrl(m)
 	containerArgs := []string{
 		"garbage-collect",
@@ -202,8 +213,8 @@ func GarbageCollectCronJob(image, clusterUid string, m v1alpha2.MondooAuditConfi
 
 		// The job runs hourly and we need to make sure that the previous one is killed before the new one is started so we don't stack them.
 		"--timeout", "55",
-		// Cleanup any resources more than 2 hours old
-		"--filter-older-than", "2h",
+		// Cleanup any resources more than 48 hours old
+		"--filter-older-than", "48h",
 		"--labels", "k8s.mondoo.com/kind=node",
 	}