From 779de48107b94863d54b366930b0aab3895ffe0b Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 8 Aug 2023 19:37:07 +0200 Subject: [PATCH 1/5] contractor: add low balance alert --- autopilot/contractor.go | 56 +++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 85751649f..ffdd24c87 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -25,7 +25,10 @@ import ( "lukechampine.com/frand" ) -var alertRenewalFailedID = frand.Entropy256() // constant until restarted +var ( + alertLowBalanceID = frand.Entropy256() // constant until restarted + alertRenewalFailedID = frand.Entropy256() // constant until restarted +) const ( // estimatedFileContractTransactionSetSize is the estimated blockchain size @@ -452,20 +455,47 @@ func (c *contractor) performWalletMaintenance(ctx context.Context) error { c.logger.Info("performing wallet maintenance") b := c.ap.bus l := c.logger + state := c.ap.State() // no contracts - nothing to do - cfg := c.ap.State().cfg - if cfg.Contracts.Amount == 0 { + if state.cfg.Contracts.Amount == 0 { l.Warn("wallet maintenance skipped, no contracts wanted") return nil } // no allowance - nothing to do - if cfg.Contracts.Allowance.IsZero() { + if state.cfg.Contracts.Allowance.IsZero() { l.Warn("wallet maintenance skipped, no allowance set") return nil } + // not enough balance - nothing to do + wallet, err := b.Wallet(ctx) + if err != nil { + l.Errorf("wallet maintenance skipped, fetching wallet balance failed with err: %v", err) + return err + } + balance := wallet.Spendable + + // register an alert if the wallet balance is low + min, max := initialContractFundingMinMax(state.cfg) + if balance.Cmp(max) <= 0 { + severity := alerts.SeverityWarning + if balance.Cmp(min) < 0 { + severity = alerts.SeverityCritical + } + c.ap.alerts.Register(alerts.Alert{ + ID: alertLowBalanceID, + Severity: severity, + Message: "wallet is low on funds", + Data: map[string]any{ + "address": state.address, + "balance": balance, + }, + Timestamp: time.Now(), + }) + } + // pending maintenance transaction - nothing to do pending, err := b.WalletPending(ctx) if err != nil { @@ -483,26 +513,20 @@ func (c *contractor) performWalletMaintenance(ctx context.Context) error { if err != nil { return err } - if uint64(len(available)) >= cfg.Contracts.Amount { - l.Debugf("no wallet maintenance needed, plenty of outputs available (%v>=%v)", len(available), cfg.Contracts.Amount) + if uint64(len(available)) >= state.cfg.Contracts.Amount { + l.Debugf("no wallet maintenance needed, plenty of outputs available (%v>=%v)", len(available), state.cfg.Contracts.Amount) return nil } - // not enough balance - nothing to do - wi, err := b.Wallet(ctx) - if err != nil { - l.Errorf("wallet maintenance skipped, fetching wallet balance failed with err: %v", err) - return err - } - balance := wi.Spendable - amount := cfg.Contracts.Allowance.Div64(cfg.Contracts.Amount) + // not enough balance to redistribute outputs - nothing to do + amount := state.cfg.Contracts.Allowance.Div64(state.cfg.Contracts.Amount) outputs := balance.Div(amount).Big().Uint64() if outputs < 2 { l.Warnf("wallet maintenance skipped, wallet has insufficient balance %v", balance) return err } - if outputs > cfg.Contracts.Amount { - outputs = cfg.Contracts.Amount + if outputs > state.cfg.Contracts.Amount { + outputs = state.cfg.Contracts.Amount } // redistribute outputs From 46e235f5df11c2f207998fa232f04ff6f13ee5da Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 8 Aug 2023 19:32:28 +0200 Subject: [PATCH 2/5] autopilot: remove errors.go --- autopilot/contractor.go | 6 +++--- autopilot/errors.go | 39 --------------------------------------- 2 files changed, 3 insertions(+), 42 deletions(-) delete mode 100644 autopilot/errors.go diff --git a/autopilot/contractor.go b/autopilot/contractor.go index ffdd24c87..b160553f0 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -1253,7 +1253,7 @@ func (c *contractor) renewContract(ctx context.Context, w Worker, ci contractInf newRevision, _, err := w.RHPRenew(ctx, fcid, endHeight, hk, contract.SiamuxAddr, settings.Address, state.address, renterFunds, newCollateral, settings.WindowSize) if err != nil { c.logger.Errorw(fmt.Sprintf("renewal failed, err: %v", err), "hk", hk, "fcid", fcid) - if containsError(err, wallet.ErrInsufficientBalance) { + if strings.Contains(err.Error(), wallet.ErrInsufficientBalance.Error()) { return api.ContractMetadata{}, false, err } return api.ContractMetadata{}, true, err @@ -1341,7 +1341,7 @@ func (c *contractor) refreshContract(ctx context.Context, w Worker, ci contractI newRevision, _, err := w.RHPRenew(ctx, contract.ID, contract.EndHeight(), hk, contract.SiamuxAddr, settings.Address, state.address, renterFunds, newCollateral, settings.WindowSize) if err != nil { c.logger.Errorw(fmt.Sprintf("refresh failed, err: %v", err), "hk", hk, "fcid", fcid) - if containsError(err, wallet.ErrInsufficientBalance) { + if strings.Contains(err.Error(), wallet.ErrInsufficientBalance.Error()) { return api.ContractMetadata{}, false, err } return api.ContractMetadata{}, true, err @@ -1413,7 +1413,7 @@ func (c *contractor) formContract(ctx context.Context, w Worker, host hostdb.Hos if err != nil { // TODO: keep track of consecutive failures and break at some point c.logger.Errorw(fmt.Sprintf("contract formation failed, err: %v", err), "hk", hk) - if containsError(err, wallet.ErrInsufficientBalance) { + if strings.Contains(err.Error(), wallet.ErrInsufficientBalance.Error()) { return api.ContractMetadata{}, false, err } return api.ContractMetadata{}, true, err diff --git a/autopilot/errors.go b/autopilot/errors.go deleted file mode 100644 index 018bd7246..000000000 --- a/autopilot/errors.go +++ /dev/null @@ -1,39 +0,0 @@ -package autopilot - -import ( - "errors" - "strings" -) - -func containsError(x, y error) bool { - return strings.Contains(x.Error(), y.Error()) -} - -func errStr(err error) string { - if err != nil { - return err.Error() - } - return "" -} - -func joinErrors(errs []error) error { - filtered := errs[:0] - for _, err := range errs { - if err != nil { - filtered = append(filtered, err) - } - } - - switch len(filtered) { - case 0: - return nil - case 1: - return filtered[0] - default: - strs := make([]string, len(filtered)) - for i := range strs { - strs[i] = filtered[i].Error() - } - return errors.New(strings.Join(strs, ";")) - } -} From 0c07a08c4f4aed36fab768a65a5a18dff041454a Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 8 Aug 2023 19:45:23 +0200 Subject: [PATCH 3/5] contractor: update comments --- autopilot/contractor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autopilot/contractor.go b/autopilot/contractor.go index b160553f0..934c0e018 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -469,7 +469,7 @@ func (c *contractor) performWalletMaintenance(ctx context.Context) error { return nil } - // not enough balance - nothing to do + // fetch wallet balance wallet, err := b.Wallet(ctx) if err != nil { l.Errorf("wallet maintenance skipped, fetching wallet balance failed with err: %v", err) @@ -477,7 +477,7 @@ func (c *contractor) performWalletMaintenance(ctx context.Context) error { } balance := wallet.Spendable - // register an alert if the wallet balance is low + // register an alert if balance is low min, max := initialContractFundingMinMax(state.cfg) if balance.Cmp(max) <= 0 { severity := alerts.SeverityWarning From 1c50f87c540bcdffb3f6d763a70bc8ec26e26173 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 9 Aug 2023 11:07:22 +0200 Subject: [PATCH 4/5] contractor: update low balance threshold --- autopilot/contractor.go | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 934c0e018..0de59bc25 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -453,37 +453,53 @@ func (c *contractor) performWalletMaintenance(ctx context.Context) error { } c.logger.Info("performing wallet maintenance") + + // convenience variables b := c.ap.bus l := c.logger state := c.ap.State() + cfg := state.cfg + period := state.period + renewWindow := cfg.Contracts.RenewWindow // no contracts - nothing to do - if state.cfg.Contracts.Amount == 0 { + if cfg.Contracts.Amount == 0 { l.Warn("wallet maintenance skipped, no contracts wanted") return nil } // no allowance - nothing to do - if state.cfg.Contracts.Allowance.IsZero() { + if cfg.Contracts.Allowance.IsZero() { l.Warn("wallet maintenance skipped, no allowance set") return nil } + // fetch consensus state + cs, err := c.ap.bus.ConsensusState(ctx) + if err != nil { + l.Warn("wallet maintenance skipped, fetching consensus state failed with err: %v", err) + return err + } + bh := cs.BlockHeight + // fetch wallet balance wallet, err := b.Wallet(ctx) if err != nil { l.Errorf("wallet maintenance skipped, fetching wallet balance failed with err: %v", err) return err } - balance := wallet.Spendable + balance := wallet.Confirmed // register an alert if balance is low - min, max := initialContractFundingMinMax(state.cfg) - if balance.Cmp(max) <= 0 { - severity := alerts.SeverityWarning - if balance.Cmp(min) < 0 { + if balance.Cmp(cfg.Contracts.Allowance) < 0 { + // increase severity as we progress through renew window + severity := alerts.SeverityInfo + if bh+renewWindow/2 >= endHeight(cfg, period) { severity = alerts.SeverityCritical + } else if bh+renewWindow >= endHeight(cfg, period) { + severity = alerts.SeverityWarning } + c.ap.alerts.Register(alerts.Alert{ ID: alertLowBalanceID, Severity: severity, @@ -513,20 +529,20 @@ func (c *contractor) performWalletMaintenance(ctx context.Context) error { if err != nil { return err } - if uint64(len(available)) >= state.cfg.Contracts.Amount { - l.Debugf("no wallet maintenance needed, plenty of outputs available (%v>=%v)", len(available), state.cfg.Contracts.Amount) + if uint64(len(available)) >= cfg.Contracts.Amount { + l.Debugf("no wallet maintenance needed, plenty of outputs available (%v>=%v)", len(available), cfg.Contracts.Amount) return nil } // not enough balance to redistribute outputs - nothing to do - amount := state.cfg.Contracts.Allowance.Div64(state.cfg.Contracts.Amount) + amount := cfg.Contracts.Allowance.Div64(cfg.Contracts.Amount) outputs := balance.Div(amount).Big().Uint64() if outputs < 2 { l.Warnf("wallet maintenance skipped, wallet has insufficient balance %v", balance) return err } - if outputs > state.cfg.Contracts.Amount { - outputs = state.cfg.Contracts.Amount + if outputs > cfg.Contracts.Amount { + outputs = cfg.Contracts.Amount } // redistribute outputs From 79b6c7f168e84198d3a161b3930f2d8ab710b105 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 9 Aug 2023 11:40:35 +0200 Subject: [PATCH 5/5] contractor: fix log method --- autopilot/contractor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 0de59bc25..2dbf3e5c8 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -477,7 +477,7 @@ func (c *contractor) performWalletMaintenance(ctx context.Context) error { // fetch consensus state cs, err := c.ap.bus.ConsensusState(ctx) if err != nil { - l.Warn("wallet maintenance skipped, fetching consensus state failed with err: %v", err) + l.Warnf("wallet maintenance skipped, fetching consensus state failed with err: %v", err) return err } bh := cs.BlockHeight @@ -485,7 +485,7 @@ func (c *contractor) performWalletMaintenance(ctx context.Context) error { // fetch wallet balance wallet, err := b.Wallet(ctx) if err != nil { - l.Errorf("wallet maintenance skipped, fetching wallet balance failed with err: %v", err) + l.Warnf("wallet maintenance skipped, fetching wallet balance failed with err: %v", err) return err } balance := wallet.Confirmed