Skip to content

Commit

Permalink
Increase readiness and liveness probe timeouts, introduce an initial …
Browse files Browse the repository at this point in the history 
…delay

Currently the default timeout of 1 second and no initial delay is applied
to the probes of the runner pods. Depending on the startup time this
can cause random Pod errors causing a whole TestRun to fail.

At some point it might also make sense to introduce a startupProbe to cover
the longer initial startup time a K6 instance / pod might need instead of ever
increasing the runtime liveness and readiness checks.

Since having the Liveness and Readiness checks be just the same makes not much
sense, as the liveness check fail will cause the container to be restarted,
this change also splits up those two tests, to allow for more individual
configuration, be it timers or what is actually checked.

Fixes grafana#306

Signed-off-by: Christian Rohmann <[email protected]>
  • Loading branch information
@frittentheke
frittentheke committed Aug 13, 2024
1 parent 40dce3f commit 0ed7572
Show file tree
Hide file tree
Showing 2 changed files with 54 additions and 3 deletions.
24 changes: 21 additions & 3 deletions pkg/resources/jobs/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,8 +185,8 @@ func NewRunnerJob(k6 v1alpha1.TestRunI, index int, token string) (*batchv1.Job,
VolumeMounts: volumeMounts,
Ports: ports,
EnvFrom: k6.GetSpec().Runner.EnvFrom,
LivenessProbe: generateProbe(k6.GetSpec().Runner.LivenessProbe),
ReadinessProbe: generateProbe(k6.GetSpec().Runner.ReadinessProbe),
LivenessProbe: generateLivenessProbe(k6.GetSpec().Runner.LivenessProbe),
ReadinessProbe: generateReadinessProbe(k6.GetSpec().Runner.ReadinessProbe),
SecurityContext: &k6.GetSpec().Runner.ContainerSecurityContext,
}},
TerminationGracePeriodSeconds: &zero,
Expand Down Expand Up @@ -276,11 +276,29 @@ func newAntiAffinity() *corev1.Affinity {
}
}

func generateProbe(configuredProbe *corev1.Probe) *corev1.Probe {
func generateLivenessProbe(configuredProbe *corev1.Probe) *corev1.Probe {
if configuredProbe != nil {
return configuredProbe
}
return &corev1.Probe{
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Port: intstr.IntOrString{IntVal: 6565},
Scheme: "HTTP",
},
},
}
}

func generateReadinessProbe(configuredProbe *corev1.Probe) *corev1.Probe {
if configuredProbe != nil {
return configuredProbe
}
return &corev1.Probe{
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand Down
33 changes: 33 additions & 0 deletions pkg/resources/jobs/runner_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,8 @@ func TestNewRunnerJob(t *testing.T) {
},
},
LivenessProbe: &corev1.Probe{
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand All @@ -348,6 +350,7 @@ func TestNewRunnerJob(t *testing.T) {
},
},
ReadinessProbe: &corev1.Probe{
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand Down Expand Up @@ -463,6 +466,8 @@ func TestNewRunnerJobNoisy(t *testing.T) {
VolumeMounts: script.VolumeMount(),
Ports: []corev1.ContainerPort{{ContainerPort: 6565}},
LivenessProbe: &corev1.Probe{
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand All @@ -472,6 +477,7 @@ func TestNewRunnerJobNoisy(t *testing.T) {
},
},
ReadinessProbe: &corev1.Probe{
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand Down Expand Up @@ -578,6 +584,8 @@ func TestNewRunnerJobUnpaused(t *testing.T) {
VolumeMounts: script.VolumeMount(),
Ports: []corev1.ContainerPort{{ContainerPort: 6565}},
LivenessProbe: &corev1.Probe{
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand All @@ -587,6 +595,7 @@ func TestNewRunnerJobUnpaused(t *testing.T) {
},
},
ReadinessProbe: &corev1.Probe{
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand Down Expand Up @@ -693,6 +702,8 @@ func TestNewRunnerJobArguments(t *testing.T) {
VolumeMounts: script.VolumeMount(),
Ports: []corev1.ContainerPort{{ContainerPort: 6565}},
LivenessProbe: &corev1.Probe{
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand All @@ -702,6 +713,7 @@ func TestNewRunnerJobArguments(t *testing.T) {
},
},
ReadinessProbe: &corev1.Probe{
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand Down Expand Up @@ -809,6 +821,8 @@ func TestNewRunnerJobServiceAccount(t *testing.T) {
VolumeMounts: script.VolumeMount(),
Ports: []corev1.ContainerPort{{ContainerPort: 6565}},
LivenessProbe: &corev1.Probe{
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand All @@ -818,6 +832,7 @@ func TestNewRunnerJobServiceAccount(t *testing.T) {
},
},
ReadinessProbe: &corev1.Probe{
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand Down Expand Up @@ -939,6 +954,8 @@ func TestNewRunnerJobIstio(t *testing.T) {
VolumeMounts: script.VolumeMount(),
Ports: []corev1.ContainerPort{{ContainerPort: 6565}},
LivenessProbe: &corev1.Probe{
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand All @@ -948,6 +965,7 @@ func TestNewRunnerJobIstio(t *testing.T) {
},
},
ReadinessProbe: &corev1.Probe{
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand Down Expand Up @@ -1064,6 +1082,8 @@ func TestNewRunnerJobCloud(t *testing.T) {
VolumeMounts: script.VolumeMount(),
Ports: []corev1.ContainerPort{{ContainerPort: 6565}},
LivenessProbe: &corev1.Probe{
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand All @@ -1073,6 +1093,7 @@ func TestNewRunnerJobCloud(t *testing.T) {
},
},
ReadinessProbe: &corev1.Probe{
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand Down Expand Up @@ -1181,6 +1202,8 @@ func TestNewRunnerJobLocalFile(t *testing.T) {
VolumeMounts: script.VolumeMount(),
Ports: []corev1.ContainerPort{{ContainerPort: 6565}},
LivenessProbe: &corev1.Probe{
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand All @@ -1190,6 +1213,7 @@ func TestNewRunnerJobLocalFile(t *testing.T) {
},
},
ReadinessProbe: &corev1.Probe{
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand Down Expand Up @@ -1323,6 +1347,8 @@ func TestNewRunnerJobWithInitContainer(t *testing.T) {
},
},
LivenessProbe: &corev1.Probe{
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand All @@ -1332,6 +1358,7 @@ func TestNewRunnerJobWithInitContainer(t *testing.T) {
},
},
ReadinessProbe: &corev1.Probe{
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand Down Expand Up @@ -1502,6 +1529,8 @@ func TestNewRunnerJobWithVolume(t *testing.T) {
},
},
LivenessProbe: &corev1.Probe{
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand All @@ -1511,6 +1540,7 @@ func TestNewRunnerJobWithVolume(t *testing.T) {
},
},
ReadinessProbe: &corev1.Probe{
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand Down Expand Up @@ -1672,6 +1702,8 @@ func TestNewRunnerJobPLZTestRun(t *testing.T) {
},
},
LivenessProbe: &corev1.Probe{
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand All @@ -1681,6 +1713,7 @@ func TestNewRunnerJobPLZTestRun(t *testing.T) {
},
},
ReadinessProbe: &corev1.Probe{
TimeoutSeconds: 3,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Expand Down

0 comments on commit 0ed7572

Please sign in to comment.