openstack: add config for retry attempts

Signed-off-by: Benny Zlotnik <[email protected]>
kubev2v · Dec 25, 2023 · b52a528 · b52a528
1 parent aa6c884
commit b52a528
Show file tree

Hide file tree

Showing 6 changed files with 18 additions and 5 deletions.
diff --git a/operator/config/manager/manager.yaml b/operator/config/manager/manager.yaml
@@ -62,6 +62,8 @@ spec:
           value: ${SNAPSHOT_REMOVAL_TIMEOUT}
         - name: SNAPSHOT_STATUS_CHECK_RATE
           value: ${SNAPSHOT_STATUS_CHECK_RATE}
+        - name: CLEANUP_RETRIES
+          value: ${CLEANUP_RETRIES}
         - name: CDI_EXPORT_TOKEN_TTL
           value: ${CDI_EXPORT_TOKEN_TTL}
         - name: FILESYSTEM_OVERHEAD

diff --git a/operator/roles/forkliftcontroller/defaults/main.yml b/operator/roles/forkliftcontroller/defaults/main.yml
@@ -31,6 +31,7 @@ controller_log_level: 3
 controller_precopy_interval: 60
 controller_snapshot_removal_timeout_minuts: 120
 controller_snapshot_status_check_rate_seconds: 10
+controller_cleanup_retries: 10
 controller_vsphere_incremental_backup: true
 controller_ovirt_warm_migration: true
 controller_max_vm_inflight: 20

diff --git a/operator/roles/forkliftcontroller/templates/controller/deployment-controller.yml.j2 b/operator/roles/forkliftcontroller/templates/controller/deployment-controller.yml.j2
@@ -68,6 +68,10 @@ spec:
         - name: SNAPSHOT_STATUS_CHECK_RATE_SECONDS
           value: "{{ controller_snapshot_status_check_rate_seconds }}"
 {% endif %}
+{% if controller_cleanup_retries is number %}
+        - name: CLEANUP_RETRIES
+          value: "{{ controller_cleanup_retries }}"
+{% endif %}
 {% if controller_max_vm_inflight is number %}
         - name: MAX_VM_INFLIGHT
           value: "{{ controller_max_vm_inflight }}"

diff --git a/pkg/controller/plan/adapter/openstack/BUILD.bazel b/pkg/controller/plan/adapter/openstack/BUILD.bazel
@@ -26,6 +26,7 @@ go_library(
         "//pkg/lib/client/openstack",
         "//pkg/lib/error",
         "//pkg/lib/itinerary",
+        "//pkg/settings",
         "//vendor/k8s.io/api/core/v1:core",
         "//vendor/k8s.io/apimachinery/pkg/api/errors",
         "//vendor/k8s.io/apimachinery/pkg/api/resource",

diff --git a/pkg/controller/plan/adapter/openstack/client.go b/pkg/controller/plan/adapter/openstack/client.go
@@ -11,6 +11,7 @@ import (
 	model "github.com/konveyor/forklift-controller/pkg/controller/provider/web/openstack"
 	libclient "github.com/konveyor/forklift-controller/pkg/lib/client/openstack"
 	liberr "github.com/konveyor/forklift-controller/pkg/lib/error"
+	"github.com/konveyor/forklift-controller/pkg/settings"
 	"k8s.io/apimachinery/pkg/util/wait"
 	cdi "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1"
 )
@@ -555,7 +556,7 @@ func (r *Client) cleanup(vm *libclient.VM, originalVolumeID string) (err error)
 		volume, err := r.getVolumeFromSnapshot(vm, snapshot.ID)
 		if err != nil {
 			if errors.Is(err, ResourceNotFoundError) {
-				r.Log.Info("volume not found, we are done")
+				r.Log.Info("volume doesn't exist, assuming we are done")
 				done = true
 				err = nil
 				return
@@ -578,19 +579,17 @@ func (r *Client) cleanup(vm *libclient.VM, originalVolumeID string) (err error)
 		return
 	}
 
-	// TODO add config
 	backoff := wait.Backoff{
 		Duration: 3 * time.Second,
-		Factor:   1.1,
-		Steps:    100,
+		Factor:   1.5,
+		Steps:    settings.Settings.CleanupRetries,
 	}
 
 	err = wait.ExponentialBackoff(backoff, condition)
 	if err != nil {
 		err = liberr.Wrap(err)
 		r.Log.Error(err, "waiting for the volume to be removed",
 			"vm", vm.Name, "snapshotID", snapshot.ID)
-		err = nil
 		return
 	}
 

diff --git a/pkg/settings/migration.go b/pkg/settings/migration.go
@@ -22,6 +22,7 @@ const (
 	CDIExportTokenTTL       = "CDI_EXPORT_TOKEN_TTL"
 	FileSystemOverhead      = "FILESYSTEM_OVERHEAD"
 	BlockOverhead           = "BLOCK_OVERHEAD"
+	CleanupRetries          = "CLEANUP_RETRIES"
 )
 
 // Migration settings
@@ -49,6 +50,8 @@ type Migration struct {
 	FileSystemOverhead int
 	// Block fixed overhead size
 	BlockOverhead int64
+	// Cleanup retries
+	CleanupRetries int
 }
 
 // Load settings.
@@ -71,6 +74,9 @@ func (r *Migration) Load() (err error) {
 	if r.SnapshotStatusCheckRate, err = getPositiveEnvLimit(SnapshotStatusCheckRate, 10); err != nil {
 		return liberr.Wrap(err)
 	}
+	if r.CleanupRetries, err = getPositiveEnvLimit(CleanupRetries, 10); err != nil {
+		return liberr.Wrap(err)
+	}
 	if virtV2vImage, ok := os.LookupEnv(VirtV2vImage); ok {
 		if cold, warm, found := strings.Cut(virtV2vImage, "|"); found {
 			r.VirtV2vImageCold = cold