From c17252a8f250cc0184918ec93dab5e7adeb31791 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 23 Feb 2024 13:14:30 +0100 Subject: [PATCH] autopilot: update churn alert to contain timestamp --- autopilot/churn.go | 41 +++++++++++++++++------------- autopilot/contractor.go | 56 +++++++++++++++++++++++++++++------------ 2 files changed, 63 insertions(+), 34 deletions(-) diff --git a/autopilot/churn.go b/autopilot/churn.go index 70c4651c2..fdc1a0f54 100644 --- a/autopilot/churn.go +++ b/autopilot/churn.go @@ -9,15 +9,15 @@ import ( type ( accumulatedChurn struct { - additions map[types.FileContractID][]contractSetAddition - removals map[types.FileContractID][]contractSetRemoval + additions map[types.FileContractID]contractSetAdditions + removals map[types.FileContractID]contractSetRemovals } ) func newAccumulatedChurn() *accumulatedChurn { return &accumulatedChurn{ - additions: make(map[types.FileContractID][]contractSetAddition), - removals: make(map[types.FileContractID][]contractSetRemoval), + additions: make(map[types.FileContractID]contractSetAdditions), + removals: make(map[types.FileContractID]contractSetRemovals), } } @@ -27,13 +27,6 @@ func (c *accumulatedChurn) Alert(name string) alerts.Alert { hint = "A high churn rate can lead to a lot of unnecessary migrations, it might be necessary to tweak your configuration depending on the reason hosts are being discarded from the set." } - removedReasons := make(map[string][]string, len(c.removals)) - for fcid, contractRemovals := range c.removals { - for _, removal := range contractRemovals { - removedReasons[fcid.String()] = append(removedReasons[fcid.String()], removal.Reason) - } - } - return alerts.Alert{ ID: alertChurnID, Severity: alerts.SeverityInfo, @@ -48,16 +41,28 @@ func (c *accumulatedChurn) Alert(name string) alerts.Alert { } } -func (c *accumulatedChurn) Apply(additions map[types.FileContractID]contractSetAddition, removals map[types.FileContractID]contractSetRemoval) { - for fcid, addition := range additions { - c.additions[fcid] = append(c.additions[fcid], addition) +func (c *accumulatedChurn) Apply(additions map[types.FileContractID]contractSetAdditions, removals map[types.FileContractID]contractSetRemovals) { + for fcid, a := range additions { + if _, exists := c.additions[fcid]; !exists { + c.additions[fcid] = a + } else { + additions := c.additions[fcid] + additions.Additions = append(additions.Additions, a.Additions...) + c.additions[fcid] = additions + } } - for fcid, removal := range removals { - c.removals[fcid] = append(c.removals[fcid], removal) + for fcid, r := range removals { + if _, exists := c.removals[fcid]; !exists { + c.removals[fcid] = r + } else { + removals := c.removals[fcid] + removals.Removals = append(removals.Removals, r.Removals...) + c.removals[fcid] = removals + } } } func (c *accumulatedChurn) Reset() { - c.additions = make(map[types.FileContractID][]contractSetAddition) - c.removals = make(map[types.FileContractID][]contractSetRemoval) + c.additions = make(map[types.FileContractID]contractSetAdditions) + c.removals = make(map[types.FileContractID]contractSetRemovals) } diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 7909277f0..9e2b52cca 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -123,15 +123,25 @@ type ( recoverable bool } + contractSetAdditions struct { + HostKey types.PublicKey `json:"hostKey"` + Additions []contractSetAddition `json:"additions"` + } + contractSetAddition struct { - Size uint64 `json:"size"` - HostKey types.PublicKey `json:"hostKey"` + Size uint64 `json:"size"` + Time api.TimeRFC3339 `json:"time"` + } + + contractSetRemovals struct { + HostKey types.PublicKey `json:"hostKey"` + Removals []contractSetRemoval `json:"removals"` } contractSetRemoval struct { - Size uint64 `json:"size"` - HostKey types.PublicKey `json:"hostKey"` - Reason string `json:"reasons"` + Size uint64 `json:"size"` + Reason string `json:"reasons"` + Time api.TimeRFC3339 `json:"time"` } renewal struct { @@ -455,8 +465,9 @@ func (c *contractor) computeContractSetChanged(ctx context.Context, name string, } // log added and removed contracts - setAdditions := make(map[types.FileContractID]contractSetAddition) - setRemovals := make(map[types.FileContractID]contractSetRemoval) + setAdditions := make(map[types.FileContractID]contractSetAdditions) + setRemovals := make(map[types.FileContractID]contractSetRemovals) + now := api.TimeNow() for _, contract := range oldSet { _, exists := inNewSet[contract.ID] _, renewed := inNewSet[renewalsFromTo[contract.ID]] @@ -466,11 +477,18 @@ func (c *contractor) computeContractSetChanged(ctx context.Context, name string, reason = "unknown" } - setRemovals[contract.ID] = contractSetRemoval{ - Size: contractData[contract.ID], - HostKey: contract.HostKey, - Reason: reason, + if _, exists := setRemovals[contract.ID]; !exists { + setRemovals[contract.ID] = contractSetRemovals{ + HostKey: contract.HostKey, + } } + removals := setRemovals[contract.ID] + removals.Removals = append(removals.Removals, contractSetRemoval{ + Size: contractData[contract.ID], + Reason: reason, + Time: now, + }) + setRemovals[contract.ID] = removals c.logger.Debugf("contract %v was removed from the contract set, size: %v, reason: %v", contract.ID, contractData[contract.ID], reason) } } @@ -478,10 +496,17 @@ func (c *contractor) computeContractSetChanged(ctx context.Context, name string, _, existed := inOldSet[contract.ID] _, renewed := renewalsToFrom[contract.ID] if !existed && !renewed { - setAdditions[contract.ID] = contractSetAddition{ - Size: contractData[contract.ID], - HostKey: contract.HostKey, + if _, exists := setAdditions[contract.ID]; !exists { + setAdditions[contract.ID] = contractSetAdditions{ + HostKey: contract.HostKey, + } } + additions := setAdditions[contract.ID] + additions.Additions = append(additions.Additions, contractSetAddition{ + Size: contractData[contract.ID], + Time: now, + }) + setAdditions[contract.ID] = additions c.logger.Debugf("contract %v was added to the contract set, size: %v", contract.ID, contractData[contract.ID]) } } @@ -501,7 +526,6 @@ func (c *contractor) computeContractSetChanged(ctx context.Context, name string, } // record churn metrics - now := api.TimeNow() var metrics []api.ContractSetChurnMetric for fcid := range setAdditions { metrics = append(metrics, api.ContractSetChurnMetric{ @@ -516,7 +540,7 @@ func (c *contractor) computeContractSetChanged(ctx context.Context, name string, Name: c.ap.state.cfg.Contracts.Set, ContractID: fcid, Direction: api.ChurnDirRemoved, - Reason: removal.Reason, + Reason: removal.Removals[0].Reason, Timestamp: now, }) }