Keep the populator pod on failure

This patch will keep the populator pod existence when failing. It won't restart or recreated. This is in the mind of aligning with v2v pod failure and the thought that once we fail, we will most likely keep failing. This change will allow to get the populator logs in case of failure in a easy way. Signed-off-by: Liran Rotenberg <[email protected]>
kubev2v · Oct 31, 2023 · 7828ad1 · 7828ad1
1 parent a55e08e
commit 7828ad1
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 10 deletions.
diff --git a/pkg/controller/plan/migration.go b/pkg/controller/plan/migration.go
@@ -1453,7 +1453,7 @@ func (r *Migration) updateCopyProgress(vm *plan.VMStatus, step *plan.Step) (err
 				if r.Plan.Spec.Warm && len(importer.Status.ContainerStatuses) > 0 {
 					vm.Warm.Failures = int(importer.Status.ContainerStatuses[0].RestartCount)
 				}
-				if RestartLimitExceeded(importer) {
+				if restartLimitExceeded(importer) {
 					task.MarkedCompleted()
 					msg, _ := terminationMessage(importer)
 					task.AddError(msg)
@@ -1598,6 +1598,10 @@ func (r *Migration) updatePopulatorCopyProgress(vm *plan.VMStatus, step *plan.St
 	if err != nil {
 		return
 	}
+	populatorPods, err := r.kubevirt.getPopulatorPods()
+	if err != nil {
+		return
+	}
 
 	for _, pvc := range pvcs {
 		if _, ok := pvc.Annotations["lun"]; ok {
@@ -1616,6 +1620,16 @@ func (r *Migration) updatePopulatorCopyProgress(vm *plan.VMStatus, step *plan.St
 			continue
 		}
 
+		for _, pod := range populatorPods {
+			pvcId := strings.Split(pod.Name, "populate-")[1]
+			if string(pvc.UID) != pvcId {
+				continue
+			}
+			if pod.Status.Phase == core.PodFailed {
+				return fmt.Errorf("populator pod %s/%s failed for PVC %s. Please check the pod logs.", pod.Namespace, pod.Name, pvcId)
+			}
+		}
+
 		if pvc.Status.Phase == core.ClaimBound {
 			task.Phase = Completed
 			task.Reason = TransferCompleted
@@ -1701,7 +1715,7 @@ func terminationMessage(pod *core.Pod) (msg string, ok bool) {
 }
 
 // Return whether the pod has failed and restarted too many times.
-func RestartLimitExceeded(pod *core.Pod) (exceeded bool) {
+func restartLimitExceeded(pod *core.Pod) (exceeded bool) {
 	if len(pod.Status.ContainerStatuses) == 0 {
 		return
 	}

diff --git a/pkg/lib-volume-populator/populator-machinery/controller.go b/pkg/lib-volume-populator/populator-machinery/controller.go
@@ -31,7 +31,6 @@ import (
 	"time"
 
 	"github.com/konveyor/forklift-controller/pkg/apis/forklift/v1beta1"
-	"github.com/konveyor/forklift-controller/pkg/controller/plan"
 	corev1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -698,13 +697,6 @@ func (c *controller) syncPvc(ctx context.Context, key, pvcNamespace, pvcName str
 		if corev1.PodSucceeded != pod.Status.Phase {
 			if corev1.PodFailed == pod.Status.Phase {
 				c.recorder.Eventf(pvc, corev1.EventTypeWarning, reasonPodFailed, "Populator failed: %s", pod.Status.Message)
-				// Delete failed pods so we can try again
-				if !plan.RestartLimitExceeded(pod) {
-					err = c.kubeClient.CoreV1().Pods(populatorNamespace).Delete(ctx, pod.Name, metav1.DeleteOptions{})
-					if err != nil {
-						return err
-					}
-				}
 			}
 			// We'll get called again later when the pod succeeds
 			return nil