Skip to content

Commit

Permalink
autopilot: address comments
Browse files Browse the repository at this point in the history
  • Loading branch information
ChrisSchinnerl committed Jul 30, 2024
1 parent 86a2bc2 commit 64575c0
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 24 deletions.
3 changes: 1 addition & 2 deletions autopilot/alerts.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,8 +170,7 @@ func newRefreshHealthFailedAlert(err error) alerts.Alert {
Severity: alerts.SeverityCritical,
Message: "Health refresh failed",
Data: map[string]interface{}{
"migrationsInterrupted": false,
"error": err.Error(),
"error": err.Error(),
},
Timestamp: time.Now(),
}
Expand Down
48 changes: 26 additions & 22 deletions autopilot/migrator.go
Original file line number Diff line number Diff line change
Expand Up @@ -206,28 +206,20 @@ func (m *migrator) performMigrations(p *workerPool) {
default:
}

OUTER:
for {
// fetch currently configured set
autopilot, err := m.ap.Config(m.ap.shutdownCtx)
if err != nil {
m.logger.Errorf("failed to fetch autopilot config: %w", err)
return
}
set := autopilot.Config.Contracts.Set
if set == "" {
m.logger.Error("could not perform migrations, no contract set configured")
return
}

// recompute health.
start := time.Now()
if err := b.RefreshHealth(m.ap.shutdownCtx); err != nil {
m.ap.RegisterAlert(m.ap.shutdownCtx, newRefreshHealthFailedAlert(err))
m.logger.Errorf("failed to recompute cached health before migration: %v", err)
}
m.logger.Infof("recomputed slab health in %v", time.Since(start))
// fetch currently configured set
autopilot, err := m.ap.Config(m.ap.shutdownCtx)
if err != nil {
m.logger.Errorf("failed to fetch autopilot config: %w", err)
return
}
set := autopilot.Config.Contracts.Set
if set == "" {
m.logger.Error("could not perform migrations, no contract set configured")
return
}

// helper to update 'toMigrate'
updateToMigrate := func() {
// fetch slabs for migration
toMigrateNew, err := b.SlabsForMigration(m.ap.shutdownCtx, m.healthCutoff, set, migratorBatchSize)
if err != nil {
Expand Down Expand Up @@ -266,7 +258,19 @@ OUTER:
sort.Slice(newSlabs, func(i, j int) bool {
return newSlabs[i].Health < newSlabs[j].Health
})
migrateNewMap = nil // free map
}

OUTER:
for {
// recompute health.
start := time.Now()
if err := b.RefreshHealth(m.ap.shutdownCtx); err != nil {
m.ap.RegisterAlert(m.ap.shutdownCtx, newRefreshHealthFailedAlert(err))
m.logger.Errorf("failed to recompute cached health before migration: %v", err)
} else {
m.logger.Infof("recomputed slab health in %v", time.Since(start))
updateToMigrate()
}

// log the updated list of slabs to migrate
m.logger.Infof("%d slabs to migrate", len(toMigrate))
Expand Down

0 comments on commit 64575c0

Please sign in to comment.