Skip to content

Commit

Permalink
autopilot: add alert hints
Browse files Browse the repository at this point in the history
  • Loading branch information
peterjan committed Oct 19, 2023
1 parent f3b762b commit 99d671f
Show file tree
Hide file tree
Showing 4 changed files with 11 additions and 8 deletions.
5 changes: 2 additions & 3 deletions autopilot/accounts.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
"go.opentelemetry.io/otel/codes"
rhpv3 "go.sia.tech/core/rhp/v3"
"go.sia.tech/core/types"
"go.sia.tech/renterd/alerts"
"go.sia.tech/renterd/api"
"go.sia.tech/renterd/tracing"
"go.uber.org/zap"
Expand Down Expand Up @@ -156,7 +155,7 @@ func (a *accounts) refillWorkerAccounts(w Worker) {
go func(contract api.ContractMetadata, inSet bool) {
rCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
accountID, refilled, rerr := refillWorkerAccount(rCtx, a.a, a.ap.bus, w, workerID, contract)
accountID, refilled, rerr := refillWorkerAccount(rCtx, a.a, w, workerID, contract)
if rerr != nil {
// register the alert on failure
a.ap.RegisterAlert(ctx, newAccountRefillAlert(accountID, contract, *rerr))
Expand Down Expand Up @@ -199,7 +198,7 @@ func (err *refillError) Is(target error) bool {
return errors.Is(err.err, target)
}

func refillWorkerAccount(ctx context.Context, a AccountStore, am alerts.Alerter, w Worker, workerID string, contract api.ContractMetadata) (accountID rhpv3.Account, refilled bool, rerr *refillError) {
func refillWorkerAccount(ctx context.Context, a AccountStore, w Worker, workerID string, contract api.ContractMetadata) (accountID rhpv3.Account, refilled bool, rerr *refillError) {
wrapErr := func(err error, keysAndValues ...interface{}) *refillError {
if err == nil {
return nil
Expand Down
10 changes: 7 additions & 3 deletions autopilot/alerts.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func (ap *Autopilot) DismissAlert(ctx context.Context, id types.Hash256) {
}
}

func newAccountLowBalanceAlert(address types.Address, balance types.Currency, bh, renewWindow, endHeight uint64) alerts.Alert {
func newAccountLowBalanceAlert(address types.Address, balance, allowance types.Currency, bh, renewWindow, endHeight uint64) alerts.Alert {
severity := alerts.SeverityInfo
if bh+renewWindow/2 >= endHeight {
severity = alerts.SeverityCritical
Expand All @@ -61,8 +61,10 @@ func newAccountLowBalanceAlert(address types.Address, balance types.Currency, bh
Severity: severity,
Message: "Wallet is low on funds",
Data: map[string]any{
"address": address,
"balance": balance,
"address": address,
"balance": balance,
"allowance": allowance,
"hint": fmt.Sprintf("The current wallet balance of %v is less than the configured allowance of %v. Ideally, a wallet holds at least one allowance worth of funds to make sure it can renew all its contracts.", balance, allowance),
},
Timestamp: time.Now(),
}
Expand Down Expand Up @@ -118,6 +120,7 @@ func newContractSetChangeAlert(name string, added, removed int, removedReasons m
"added": added,
"removed": removed,
"removals": removedReasons,
"hint": "A high churn rate can lead to a lot of unnecessary migrations, it might be necessary to tweak your configuration depending on the reason hosts are being discarded from the set.",
},
Timestamp: time.Now(),
}
Expand Down Expand Up @@ -146,6 +149,7 @@ func newSlabMigrationFailedAlert(slab object.Slab, health float64, err error) al
"error": err,
"health": health,
"slabKey": slab.Key.String(),
"hint": "Migration failures can be temporary, but if they persist it can eventually lead to data loss and should therefor be taken very seriously.",
},
Timestamp: time.Now(),
}
Expand Down
2 changes: 1 addition & 1 deletion autopilot/contractor.go
Original file line number Diff line number Diff line change
Expand Up @@ -509,7 +509,7 @@ func (c *contractor) performWalletMaintenance(ctx context.Context) error {

// register an alert if balance is low
if balance.Cmp(cfg.Contracts.Allowance) < 0 {
c.ap.RegisterAlert(ctx, newAccountLowBalanceAlert(state.address, balance, cs.BlockHeight, renewWindow, endHeight(cfg, period)))
c.ap.RegisterAlert(ctx, newAccountLowBalanceAlert(state.address, balance, cfg.Contracts.Allowance, cs.BlockHeight, renewWindow, endHeight(cfg, period)))
} else {
c.ap.DismissAlert(ctx, alertLowBalanceID)
}
Expand Down
2 changes: 1 addition & 1 deletion worker/download.go
Original file line number Diff line number Diff line change
Expand Up @@ -1098,7 +1098,7 @@ func (s *slabDownload) finish() ([][]byte, error) {
unused++
}
}
return nil, fmt.Errorf("failed to download slab: completed=%d, inflight=%d, launched=%d downloaders=%d unused=%d errors=%w", s.numCompleted, s.numInflight, s.numLaunched, s.mgr.numDownloaders(), unused, s.errs)
return nil, fmt.Errorf("failed to download slab: completed=%d, inflight=%d, launched=%d downloaders=%d unused=%d errors=%d %w", s.numCompleted, s.numInflight, s.numLaunched, s.mgr.numDownloaders(), unused, len(s.errs), s.errs)
}
return s.sectors, nil
}
Expand Down

0 comments on commit 99d671f

Please sign in to comment.