From 0ed75725075512d7daf7c638e8ba7c7c3447b4dd Mon Sep 17 00:00:00 2001 From: Christian Rohmann Date: Mon, 12 Aug 2024 17:42:55 +0200 Subject: [PATCH] Increase readiness and liveness probe timeouts, introduce an initial delay Currently the default timeout of 1 second and no initial delay is applied to the probes of the runner pods. Depending on the startup time this can cause random Pod errors causing a whole TestRun to fail. At some point it might also make sense to introduce a startupProbe to cover the longer initial startup time a K6 instance / pod might need instead of ever increasing the runtime liveness and readiness checks. Since having the Liveness and Readiness checks be just the same makes not much sense, as the liveness check fail will cause the container to be restarted, this change also splits up those two tests, to allow for more individual configuration, be it timers or what is actually checked. Fixes #306 Signed-off-by: Christian Rohmann --- pkg/resources/jobs/runner.go | 24 +++++++++++++++++++--- pkg/resources/jobs/runner_test.go | 33 +++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/pkg/resources/jobs/runner.go b/pkg/resources/jobs/runner.go index 54105401..ec2c9bb7 100644 --- a/pkg/resources/jobs/runner.go +++ b/pkg/resources/jobs/runner.go @@ -185,8 +185,8 @@ func NewRunnerJob(k6 v1alpha1.TestRunI, index int, token string) (*batchv1.Job, VolumeMounts: volumeMounts, Ports: ports, EnvFrom: k6.GetSpec().Runner.EnvFrom, - LivenessProbe: generateProbe(k6.GetSpec().Runner.LivenessProbe), - ReadinessProbe: generateProbe(k6.GetSpec().Runner.ReadinessProbe), + LivenessProbe: generateLivenessProbe(k6.GetSpec().Runner.LivenessProbe), + ReadinessProbe: generateReadinessProbe(k6.GetSpec().Runner.ReadinessProbe), SecurityContext: &k6.GetSpec().Runner.ContainerSecurityContext, }}, TerminationGracePeriodSeconds: &zero, @@ -276,11 +276,29 @@ func newAntiAffinity() *corev1.Affinity { } } -func generateProbe(configuredProbe *corev1.Probe) *corev1.Probe { +func generateLivenessProbe(configuredProbe *corev1.Probe) *corev1.Probe { if configuredProbe != nil { return configuredProbe } return &corev1.Probe{ + InitialDelaySeconds: 10, + TimeoutSeconds: 3, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/v1/status", + Port: intstr.IntOrString{IntVal: 6565}, + Scheme: "HTTP", + }, + }, + } +} + +func generateReadinessProbe(configuredProbe *corev1.Probe) *corev1.Probe { + if configuredProbe != nil { + return configuredProbe + } + return &corev1.Probe{ + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", diff --git a/pkg/resources/jobs/runner_test.go b/pkg/resources/jobs/runner_test.go index 22078cf5..72958f9c 100644 --- a/pkg/resources/jobs/runner_test.go +++ b/pkg/resources/jobs/runner_test.go @@ -339,6 +339,8 @@ func TestNewRunnerJob(t *testing.T) { }, }, LivenessProbe: &corev1.Probe{ + InitialDelaySeconds: 10, + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -348,6 +350,7 @@ func TestNewRunnerJob(t *testing.T) { }, }, ReadinessProbe: &corev1.Probe{ + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -463,6 +466,8 @@ func TestNewRunnerJobNoisy(t *testing.T) { VolumeMounts: script.VolumeMount(), Ports: []corev1.ContainerPort{{ContainerPort: 6565}}, LivenessProbe: &corev1.Probe{ + InitialDelaySeconds: 10, + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -472,6 +477,7 @@ func TestNewRunnerJobNoisy(t *testing.T) { }, }, ReadinessProbe: &corev1.Probe{ + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -578,6 +584,8 @@ func TestNewRunnerJobUnpaused(t *testing.T) { VolumeMounts: script.VolumeMount(), Ports: []corev1.ContainerPort{{ContainerPort: 6565}}, LivenessProbe: &corev1.Probe{ + InitialDelaySeconds: 10, + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -587,6 +595,7 @@ func TestNewRunnerJobUnpaused(t *testing.T) { }, }, ReadinessProbe: &corev1.Probe{ + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -693,6 +702,8 @@ func TestNewRunnerJobArguments(t *testing.T) { VolumeMounts: script.VolumeMount(), Ports: []corev1.ContainerPort{{ContainerPort: 6565}}, LivenessProbe: &corev1.Probe{ + InitialDelaySeconds: 10, + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -702,6 +713,7 @@ func TestNewRunnerJobArguments(t *testing.T) { }, }, ReadinessProbe: &corev1.Probe{ + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -809,6 +821,8 @@ func TestNewRunnerJobServiceAccount(t *testing.T) { VolumeMounts: script.VolumeMount(), Ports: []corev1.ContainerPort{{ContainerPort: 6565}}, LivenessProbe: &corev1.Probe{ + InitialDelaySeconds: 10, + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -818,6 +832,7 @@ func TestNewRunnerJobServiceAccount(t *testing.T) { }, }, ReadinessProbe: &corev1.Probe{ + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -939,6 +954,8 @@ func TestNewRunnerJobIstio(t *testing.T) { VolumeMounts: script.VolumeMount(), Ports: []corev1.ContainerPort{{ContainerPort: 6565}}, LivenessProbe: &corev1.Probe{ + InitialDelaySeconds: 10, + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -948,6 +965,7 @@ func TestNewRunnerJobIstio(t *testing.T) { }, }, ReadinessProbe: &corev1.Probe{ + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -1064,6 +1082,8 @@ func TestNewRunnerJobCloud(t *testing.T) { VolumeMounts: script.VolumeMount(), Ports: []corev1.ContainerPort{{ContainerPort: 6565}}, LivenessProbe: &corev1.Probe{ + InitialDelaySeconds: 10, + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -1073,6 +1093,7 @@ func TestNewRunnerJobCloud(t *testing.T) { }, }, ReadinessProbe: &corev1.Probe{ + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -1181,6 +1202,8 @@ func TestNewRunnerJobLocalFile(t *testing.T) { VolumeMounts: script.VolumeMount(), Ports: []corev1.ContainerPort{{ContainerPort: 6565}}, LivenessProbe: &corev1.Probe{ + InitialDelaySeconds: 10, + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -1190,6 +1213,7 @@ func TestNewRunnerJobLocalFile(t *testing.T) { }, }, ReadinessProbe: &corev1.Probe{ + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -1323,6 +1347,8 @@ func TestNewRunnerJobWithInitContainer(t *testing.T) { }, }, LivenessProbe: &corev1.Probe{ + InitialDelaySeconds: 10, + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -1332,6 +1358,7 @@ func TestNewRunnerJobWithInitContainer(t *testing.T) { }, }, ReadinessProbe: &corev1.Probe{ + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -1502,6 +1529,8 @@ func TestNewRunnerJobWithVolume(t *testing.T) { }, }, LivenessProbe: &corev1.Probe{ + InitialDelaySeconds: 10, + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -1511,6 +1540,7 @@ func TestNewRunnerJobWithVolume(t *testing.T) { }, }, ReadinessProbe: &corev1.Probe{ + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -1672,6 +1702,8 @@ func TestNewRunnerJobPLZTestRun(t *testing.T) { }, }, LivenessProbe: &corev1.Probe{ + InitialDelaySeconds: 10, + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status", @@ -1681,6 +1713,7 @@ func TestNewRunnerJobPLZTestRun(t *testing.T) { }, }, ReadinessProbe: &corev1.Probe{ + TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/v1/status",