From 278276b652c0b1a8ac28d0f6c61944040c8e5ab5 Mon Sep 17 00:00:00 2001 From: Jan Grant Date: Fri, 7 Jun 2024 15:57:32 +0100 Subject: [PATCH] e2e: wait for the provisioner to clean up PV resources If we're unlucky, the local-path-provisioner can be killed before it has a chance to clean up orphened PVs. These may then hang around while subsequent tests run. Depending on the lexical sort order of those PVs, the `typeCheckCmd` in `runTest` may select a moribund PV, causing the test to randomly fail. --- test/pod_test.go | 2 ++ .../custom-path-pattern/kustomization.yaml | 4 +-- .../labelled-deploy/kustomization.yaml | 6 ++++ .../kustomization.yaml | 2 +- .../kustomization.yaml | 2 +- .../pod-with-local-volume/kustomization.yaml | 2 +- .../pod-with-node-affinity/kustomization.yaml | 2 +- .../pod-with-rwop-volume/kustomization.yaml | 2 +- .../kustomization.yaml | 2 +- .../pod-with-subpath/kustomization.yaml | 2 +- test/testdata/pod/kustomization.yaml | 2 +- test/util.go | 30 ++++++++++++++----- 12 files changed, 40 insertions(+), 18 deletions(-) create mode 100644 test/testdata/labelled-deploy/kustomization.yaml diff --git a/test/pod_test.go b/test/pod_test.go index 5a69d7e7e..19782feca 100644 --- a/test/pod_test.go +++ b/test/pod_test.go @@ -39,6 +39,8 @@ func (p *PodTestSuite) SetupSuite() { cmds := []string{ fmt.Sprintf("kind create cluster --config=%s --wait=120s", testdataFile("kind-cluster.yaml")), fmt.Sprintf("kind load docker-image %s", p.config.IMAGE), + // We'll be replacing this, but applying particular labels to it. + "kubectl -n local-path-storage delete deployment local-path-provisioner --wait", } for _, cmd := range cmds { _, err = runCmd( diff --git a/test/testdata/custom-path-pattern/kustomization.yaml b/test/testdata/custom-path-pattern/kustomization.yaml index df458be4f..b19eddadb 100644 --- a/test/testdata/custom-path-pattern/kustomization.yaml +++ b/test/testdata/custom-path-pattern/kustomization.yaml @@ -1,7 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ../../../deploy +- ../labelled-deploy - storage-class.yaml - pod.yaml - pvc.yaml @@ -11,4 +11,4 @@ commonLabels: app: local-path-provisioner images: - name: rancher/local-path-provisioner - newTag: dev \ No newline at end of file + newTag: dev diff --git a/test/testdata/labelled-deploy/kustomization.yaml b/test/testdata/labelled-deploy/kustomization.yaml new file mode 100644 index 000000000..3891d43f6 --- /dev/null +++ b/test/testdata/labelled-deploy/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../../deploy +commonLabels: + system-component: "true" diff --git a/test/testdata/multiple-storage-classes/kustomization.yaml b/test/testdata/multiple-storage-classes/kustomization.yaml index 6efccacb7..f4287a2b0 100644 --- a/test/testdata/multiple-storage-classes/kustomization.yaml +++ b/test/testdata/multiple-storage-classes/kustomization.yaml @@ -1,7 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ../../../deploy +- ../labelled-deploy - storage-class-shared.yaml - pod.yaml - pvc.yaml diff --git a/test/testdata/pod-with-default-local-volume/kustomization.yaml b/test/testdata/pod-with-default-local-volume/kustomization.yaml index 43eb7c040..49cbec367 100644 --- a/test/testdata/pod-with-default-local-volume/kustomization.yaml +++ b/test/testdata/pod-with-default-local-volume/kustomization.yaml @@ -1,7 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ../../../deploy +- ../labelled-deploy - ../../../examples/pod patches: - path: patch.yaml diff --git a/test/testdata/pod-with-local-volume/kustomization.yaml b/test/testdata/pod-with-local-volume/kustomization.yaml index 486f32d25..31251dfe4 100644 --- a/test/testdata/pod-with-local-volume/kustomization.yaml +++ b/test/testdata/pod-with-local-volume/kustomization.yaml @@ -1,7 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ../../../deploy +- ../labelled-deploy - ../../../examples/pod-with-local-volume commonLabels: app: local-path-provisioner diff --git a/test/testdata/pod-with-node-affinity/kustomization.yaml b/test/testdata/pod-with-node-affinity/kustomization.yaml index d2b22ea85..5f9da0903 100644 --- a/test/testdata/pod-with-node-affinity/kustomization.yaml +++ b/test/testdata/pod-with-node-affinity/kustomization.yaml @@ -1,7 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ../../../deploy +- ../labelled-deploy - ../../../examples/pod-with-node-affinity patches: - path: patch.yaml diff --git a/test/testdata/pod-with-rwop-volume/kustomization.yaml b/test/testdata/pod-with-rwop-volume/kustomization.yaml index d650c9856..e1b622503 100644 --- a/test/testdata/pod-with-rwop-volume/kustomization.yaml +++ b/test/testdata/pod-with-rwop-volume/kustomization.yaml @@ -1,7 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ../../../deploy +- ../labelled-deploy - ../../../examples/pod-with-rwop-volume commonLabels: app: local-path-provisioner diff --git a/test/testdata/pod-with-security-context/kustomization.yaml b/test/testdata/pod-with-security-context/kustomization.yaml index 9dce4e585..d30919dd2 100644 --- a/test/testdata/pod-with-security-context/kustomization.yaml +++ b/test/testdata/pod-with-security-context/kustomization.yaml @@ -1,7 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ../../../deploy +- ../labelled-deploy - ../../../examples/pod-with-security-context commonLabels: app: local-path-provisioner diff --git a/test/testdata/pod-with-subpath/kustomization.yaml b/test/testdata/pod-with-subpath/kustomization.yaml index 50c587a38..f0a99f32e 100644 --- a/test/testdata/pod-with-subpath/kustomization.yaml +++ b/test/testdata/pod-with-subpath/kustomization.yaml @@ -1,7 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ../../../deploy +- ../labelled-deploy - ../../../examples/pod-with-subpath commonLabels: app: local-path-provisioner diff --git a/test/testdata/pod/kustomization.yaml b/test/testdata/pod/kustomization.yaml index e3947b9c1..c0f2cb58d 100644 --- a/test/testdata/pod/kustomization.yaml +++ b/test/testdata/pod/kustomization.yaml @@ -1,7 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ../../../deploy +- ../labelled-deploy - ../../../examples/pod commonLabels: app: local-path-provisioner diff --git a/test/util.go b/test/util.go index 7aedec0b6..3d6788242 100644 --- a/test/util.go +++ b/test/util.go @@ -76,14 +76,28 @@ func testdataFile(fields ...string) string { } func deleteKustomizeDeployment(t *testing.T, kustomizeDir string, envs []string) error { - _, err := runCmd( - t, - "kustomize build | kubectl delete --timeout=180s -f -", - testdataFile(kustomizeDir), - envs, - nil, - ) - return err + cmds := []string{ + // Begin by deleting everything except the provisioner. + // This gives the local-path provider a chance to kick in and kill the pv. + "kustomize build | kubectl delete --timeout=180s -f - -l 'system-component!=true'", + // Wait on the provisioner + "kubectl wait --for=delete pv --all", + // Clean up everything else + "kustomize build | kubectl delete --timeout=180s -f - -l 'system-component=true' --wait", + } + for _, cmd := range cmds { + _, err := runCmd( + t, + cmd, + testdataFile(kustomizeDir), + envs, + nil, + ) + if err != nil { + return err + } + } + return nil } func deleteCluster(t *testing.T, envs []string) error {