diff --git a/test/pod_test.go b/test/pod_test.go
index 5a69d7e7..19782fec 100644
--- a/test/pod_test.go
+++ b/test/pod_test.go
@@ -39,6 +39,8 @@ func (p *PodTestSuite) SetupSuite() {
 	cmds := []string{
 		fmt.Sprintf("kind create cluster --config=%s --wait=120s", testdataFile("kind-cluster.yaml")),
 		fmt.Sprintf("kind load docker-image %s", p.config.IMAGE),
+		// We'll be replacing this, but applying particular labels to it.
+		"kubectl -n local-path-storage delete deployment local-path-provisioner --wait",
 	}
 	for _, cmd := range cmds {
 		_, err = runCmd(
diff --git a/test/testdata/custom-path-pattern/kustomization.yaml b/test/testdata/custom-path-pattern/kustomization.yaml
index df458be4..b19eddad 100644
--- a/test/testdata/custom-path-pattern/kustomization.yaml
+++ b/test/testdata/custom-path-pattern/kustomization.yaml
@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- ../../../deploy
+- ../labelled-deploy
 - storage-class.yaml
 - pod.yaml
 - pvc.yaml
@@ -11,4 +11,4 @@ commonLabels:
   app: local-path-provisioner
 images:
 - name: rancher/local-path-provisioner
-  newTag: dev
\ No newline at end of file
+  newTag: dev
diff --git a/test/testdata/labelled-deploy/kustomization.yaml b/test/testdata/labelled-deploy/kustomization.yaml
new file mode 100644
index 00000000..3891d43f
--- /dev/null
+++ b/test/testdata/labelled-deploy/kustomization.yaml
@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../../deploy
+commonLabels:
+  system-component: "true"
diff --git a/test/testdata/multiple-storage-classes/kustomization.yaml b/test/testdata/multiple-storage-classes/kustomization.yaml
index 6efccacb..f4287a2b 100644
--- a/test/testdata/multiple-storage-classes/kustomization.yaml
+++ b/test/testdata/multiple-storage-classes/kustomization.yaml
@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- ../../../deploy
+- ../labelled-deploy
 - storage-class-shared.yaml
 - pod.yaml
 - pvc.yaml
diff --git a/test/testdata/pod-with-default-local-volume/kustomization.yaml b/test/testdata/pod-with-default-local-volume/kustomization.yaml
index 43eb7c04..49cbec36 100644
--- a/test/testdata/pod-with-default-local-volume/kustomization.yaml
+++ b/test/testdata/pod-with-default-local-volume/kustomization.yaml
@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- ../../../deploy
+- ../labelled-deploy
 - ../../../examples/pod
 patches:
 - path: patch.yaml
diff --git a/test/testdata/pod-with-local-volume/kustomization.yaml b/test/testdata/pod-with-local-volume/kustomization.yaml
index 486f32d2..31251dfe 100644
--- a/test/testdata/pod-with-local-volume/kustomization.yaml
+++ b/test/testdata/pod-with-local-volume/kustomization.yaml
@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- ../../../deploy
+- ../labelled-deploy
 - ../../../examples/pod-with-local-volume
 commonLabels:
   app: local-path-provisioner
diff --git a/test/testdata/pod-with-node-affinity/kustomization.yaml b/test/testdata/pod-with-node-affinity/kustomization.yaml
index d2b22ea8..5f9da090 100644
--- a/test/testdata/pod-with-node-affinity/kustomization.yaml
+++ b/test/testdata/pod-with-node-affinity/kustomization.yaml
@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- ../../../deploy
+- ../labelled-deploy
 - ../../../examples/pod-with-node-affinity
 patches:
 - path: patch.yaml
diff --git a/test/testdata/pod-with-rwop-volume/kustomization.yaml b/test/testdata/pod-with-rwop-volume/kustomization.yaml
index d650c985..e1b62250 100644
--- a/test/testdata/pod-with-rwop-volume/kustomization.yaml
+++ b/test/testdata/pod-with-rwop-volume/kustomization.yaml
@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- ../../../deploy
+- ../labelled-deploy
 - ../../../examples/pod-with-rwop-volume
 commonLabels:
   app: local-path-provisioner
diff --git a/test/testdata/pod-with-security-context/kustomization.yaml b/test/testdata/pod-with-security-context/kustomization.yaml
index 9dce4e58..d30919dd 100644
--- a/test/testdata/pod-with-security-context/kustomization.yaml
+++ b/test/testdata/pod-with-security-context/kustomization.yaml
@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- ../../../deploy
+- ../labelled-deploy
 - ../../../examples/pod-with-security-context
 commonLabels:
   app: local-path-provisioner
diff --git a/test/testdata/pod-with-subpath/kustomization.yaml b/test/testdata/pod-with-subpath/kustomization.yaml
index 50c587a3..f0a99f32 100644
--- a/test/testdata/pod-with-subpath/kustomization.yaml
+++ b/test/testdata/pod-with-subpath/kustomization.yaml
@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- ../../../deploy
+- ../labelled-deploy
 - ../../../examples/pod-with-subpath
 commonLabels:
   app: local-path-provisioner
diff --git a/test/testdata/pod/kustomization.yaml b/test/testdata/pod/kustomization.yaml
index e3947b9c..c0f2cb58 100644
--- a/test/testdata/pod/kustomization.yaml
+++ b/test/testdata/pod/kustomization.yaml
@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- ../../../deploy
+- ../labelled-deploy
 - ../../../examples/pod
 commonLabels:
   app: local-path-provisioner
diff --git a/test/util.go b/test/util.go
index 7aedec0b..3d678824 100644
--- a/test/util.go
+++ b/test/util.go
@@ -76,14 +76,28 @@ func testdataFile(fields ...string) string {
 }
 
 func deleteKustomizeDeployment(t *testing.T, kustomizeDir string, envs []string) error {
-	_, err := runCmd(
-		t,
-		"kustomize build | kubectl delete --timeout=180s -f -",
-		testdataFile(kustomizeDir),
-		envs,
-		nil,
-	)
-	return err
+	cmds := []string{
+		// Begin by deleting everything except the provisioner.
+		// This gives the local-path provider a chance to kick in and kill the pv.
+		"kustomize build | kubectl delete --timeout=180s -f - -l 'system-component!=true'",
+		// Wait on the provisioner
+		"kubectl wait --for=delete pv --all",
+		// Clean up everything else
+		"kustomize build | kubectl delete --timeout=180s -f -  -l 'system-component=true' --wait",
+	}
+	for _, cmd := range cmds {
+		_, err := runCmd(
+			t,
+			cmd,
+			testdataFile(kustomizeDir),
+			envs,
+			nil,
+		)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 func deleteCluster(t *testing.T, envs []string) error {