From b52a52844d5e069a341e30200122cac9240adf5f Mon Sep 17 00:00:00 2001 From: Benny Zlotnik Date: Mon, 25 Dec 2023 11:58:29 +0200 Subject: [PATCH] openstack: add config for retry attempts Signed-off-by: Benny Zlotnik --- operator/config/manager/manager.yaml | 2 ++ operator/roles/forkliftcontroller/defaults/main.yml | 1 + .../templates/controller/deployment-controller.yml.j2 | 4 ++++ pkg/controller/plan/adapter/openstack/BUILD.bazel | 1 + pkg/controller/plan/adapter/openstack/client.go | 9 ++++----- pkg/settings/migration.go | 6 ++++++ 6 files changed, 18 insertions(+), 5 deletions(-) diff --git a/operator/config/manager/manager.yaml b/operator/config/manager/manager.yaml index d218e6335..182fa586f 100644 --- a/operator/config/manager/manager.yaml +++ b/operator/config/manager/manager.yaml @@ -62,6 +62,8 @@ spec: value: ${SNAPSHOT_REMOVAL_TIMEOUT} - name: SNAPSHOT_STATUS_CHECK_RATE value: ${SNAPSHOT_STATUS_CHECK_RATE} + - name: CLEANUP_RETRIES + value: ${CLEANUP_RETRIES} - name: CDI_EXPORT_TOKEN_TTL value: ${CDI_EXPORT_TOKEN_TTL} - name: FILESYSTEM_OVERHEAD diff --git a/operator/roles/forkliftcontroller/defaults/main.yml b/operator/roles/forkliftcontroller/defaults/main.yml index 865cb821b..e99e67ed7 100644 --- a/operator/roles/forkliftcontroller/defaults/main.yml +++ b/operator/roles/forkliftcontroller/defaults/main.yml @@ -31,6 +31,7 @@ controller_log_level: 3 controller_precopy_interval: 60 controller_snapshot_removal_timeout_minuts: 120 controller_snapshot_status_check_rate_seconds: 10 +controller_cleanup_retries: 10 controller_vsphere_incremental_backup: true controller_ovirt_warm_migration: true controller_max_vm_inflight: 20 diff --git a/operator/roles/forkliftcontroller/templates/controller/deployment-controller.yml.j2 b/operator/roles/forkliftcontroller/templates/controller/deployment-controller.yml.j2 index f34e969b3..2f04178ba 100644 --- a/operator/roles/forkliftcontroller/templates/controller/deployment-controller.yml.j2 +++ b/operator/roles/forkliftcontroller/templates/controller/deployment-controller.yml.j2 @@ -68,6 +68,10 @@ spec: - name: SNAPSHOT_STATUS_CHECK_RATE_SECONDS value: "{{ controller_snapshot_status_check_rate_seconds }}" {% endif %} +{% if controller_cleanup_retries is number %} + - name: CLEANUP_RETRIES + value: "{{ controller_cleanup_retries }}" +{% endif %} {% if controller_max_vm_inflight is number %} - name: MAX_VM_INFLIGHT value: "{{ controller_max_vm_inflight }}" diff --git a/pkg/controller/plan/adapter/openstack/BUILD.bazel b/pkg/controller/plan/adapter/openstack/BUILD.bazel index 4858b5486..8c94d5f65 100644 --- a/pkg/controller/plan/adapter/openstack/BUILD.bazel +++ b/pkg/controller/plan/adapter/openstack/BUILD.bazel @@ -26,6 +26,7 @@ go_library( "//pkg/lib/client/openstack", "//pkg/lib/error", "//pkg/lib/itinerary", + "//pkg/settings", "//vendor/k8s.io/api/core/v1:core", "//vendor/k8s.io/apimachinery/pkg/api/errors", "//vendor/k8s.io/apimachinery/pkg/api/resource", diff --git a/pkg/controller/plan/adapter/openstack/client.go b/pkg/controller/plan/adapter/openstack/client.go index 8a4e0d0f7..2c4ccdbfb 100644 --- a/pkg/controller/plan/adapter/openstack/client.go +++ b/pkg/controller/plan/adapter/openstack/client.go @@ -11,6 +11,7 @@ import ( model "github.com/konveyor/forklift-controller/pkg/controller/provider/web/openstack" libclient "github.com/konveyor/forklift-controller/pkg/lib/client/openstack" liberr "github.com/konveyor/forklift-controller/pkg/lib/error" + "github.com/konveyor/forklift-controller/pkg/settings" "k8s.io/apimachinery/pkg/util/wait" cdi "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" ) @@ -555,7 +556,7 @@ func (r *Client) cleanup(vm *libclient.VM, originalVolumeID string) (err error) volume, err := r.getVolumeFromSnapshot(vm, snapshot.ID) if err != nil { if errors.Is(err, ResourceNotFoundError) { - r.Log.Info("volume not found, we are done") + r.Log.Info("volume doesn't exist, assuming we are done") done = true err = nil return @@ -578,11 +579,10 @@ func (r *Client) cleanup(vm *libclient.VM, originalVolumeID string) (err error) return } - // TODO add config backoff := wait.Backoff{ Duration: 3 * time.Second, - Factor: 1.1, - Steps: 100, + Factor: 1.5, + Steps: settings.Settings.CleanupRetries, } err = wait.ExponentialBackoff(backoff, condition) @@ -590,7 +590,6 @@ func (r *Client) cleanup(vm *libclient.VM, originalVolumeID string) (err error) err = liberr.Wrap(err) r.Log.Error(err, "waiting for the volume to be removed", "vm", vm.Name, "snapshotID", snapshot.ID) - err = nil return } diff --git a/pkg/settings/migration.go b/pkg/settings/migration.go index 419eccd42..30b733464 100644 --- a/pkg/settings/migration.go +++ b/pkg/settings/migration.go @@ -22,6 +22,7 @@ const ( CDIExportTokenTTL = "CDI_EXPORT_TOKEN_TTL" FileSystemOverhead = "FILESYSTEM_OVERHEAD" BlockOverhead = "BLOCK_OVERHEAD" + CleanupRetries = "CLEANUP_RETRIES" ) // Migration settings @@ -49,6 +50,8 @@ type Migration struct { FileSystemOverhead int // Block fixed overhead size BlockOverhead int64 + // Cleanup retries + CleanupRetries int } // Load settings. @@ -71,6 +74,9 @@ func (r *Migration) Load() (err error) { if r.SnapshotStatusCheckRate, err = getPositiveEnvLimit(SnapshotStatusCheckRate, 10); err != nil { return liberr.Wrap(err) } + if r.CleanupRetries, err = getPositiveEnvLimit(CleanupRetries, 10); err != nil { + return liberr.Wrap(err) + } if virtV2vImage, ok := os.LookupEnv(VirtV2vImage); ok { if cold, warm, found := strings.Cut(virtV2vImage, "|"); found { r.VirtV2vImageCold = cold