Increase readiness and liveness probe timeouts, introduce an initial …

…delay

Currently the default timeout of 1 second and no initial delay is applied
to the probes of the runner pods. Depending on the startup time this
can cause random Pod errors causing a whole TestRun to fail.

At some point it might also make sense to introduce a startupProbe to cover
the longer initial startup time a K6 instance / pod might need instead of ever
increasing the runtime liveness and readiness checks.

Since having the Liveness and Readiness checks be just the same makes not much
sense, as the liveness check fail will cause the container to be restarted,
this change also splits up those two tests, to allow for more individual
configuration, be it timers or what is actually checked.

Fixes grafana#306

Signed-off-by: Christian Rohmann <[email protected]>

pkg/resources/jobs/runner.go

@@ -185,8 +185,8 @@ func NewRunnerJob(k6 v1alpha1.TestRunI, index int, token string) (*batchv1.Job,
 						VolumeMounts:    volumeMounts,
 						Ports:           ports,
 						EnvFrom:         k6.GetSpec().Runner.EnvFrom,
-						LivenessProbe:   generateProbe(k6.GetSpec().Runner.LivenessProbe),
-						ReadinessProbe:  generateProbe(k6.GetSpec().Runner.ReadinessProbe),
+						LivenessProbe:   generateLivenessProbe(k6.GetSpec().Runner.LivenessProbe),
+						ReadinessProbe:  generateReadinessProbe(k6.GetSpec().Runner.ReadinessProbe),
 						SecurityContext: &k6.GetSpec().Runner.ContainerSecurityContext,
 					}},
 					TerminationGracePeriodSeconds: &zero,
@@ -276,11 +276,29 @@ func newAntiAffinity() *corev1.Affinity {
 	}
 }
 
-func generateProbe(configuredProbe *corev1.Probe) *corev1.Probe {
+func generateLivenessProbe(configuredProbe *corev1.Probe) *corev1.Probe {
 	if configuredProbe != nil {
 		return configuredProbe
 	}
 	return &corev1.Probe{
+		InitialDelaySeconds: 10,
+		TimeoutSeconds:      3,
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/v1/status",
+				Port:   intstr.IntOrString{IntVal: 6565},
+				Scheme: "HTTP",
+			},
+		},
+	}
+}
+
+func generateReadinessProbe(configuredProbe *corev1.Probe) *corev1.Probe {
+	if configuredProbe != nil {
+		return configuredProbe
+	}
+	return &corev1.Probe{
+		TimeoutSeconds: 3,
 		ProbeHandler: corev1.ProbeHandler{
 			HTTPGet: &corev1.HTTPGetAction{
 				Path:   "/v1/status",

pkg/resources/jobs/runner_test.go

@@ -339,6 +339,8 @@ func TestNewRunnerJob(t *testing.T) {
 							},
 						},
 						LivenessProbe: &corev1.Probe{
+							InitialDelaySeconds: 10,
+							TimeoutSeconds:      3,
 							ProbeHandler: corev1.ProbeHandler{
 								HTTPGet: &corev1.HTTPGetAction{
 									Path:   "/v1/status",
@@ -348,6 +350,7 @@ func TestNewRunnerJob(t *testing.T) {
 							},
 						},
 						ReadinessProbe: &corev1.Probe{
+							TimeoutSeconds: 3,
 							ProbeHandler: corev1.ProbeHandler{
 								HTTPGet: &corev1.HTTPGetAction{
 									Path:   "/v1/status",