Merge pull request #1003 from SiaFoundation/pj/merge-dev

Merge its-happening <- dev

.github/workflows/test.yml

@@ -47,6 +47,16 @@ jobs:
         uses: n8maninger/action-golang-test@v1
         with:
           args: "-race;-short"
+      - name: Test Stores - MySQL
+        if: matrix.os == 'ubuntu-latest'
+        uses: n8maninger/action-golang-test@v1
+        env:
+          RENTERD_DB_URI: 127.0.0.1:3800
+          RENTERD_DB_USER: root
+          RENTERD_DB_PASSWORD: test
+        with:
+          package: "./stores"
+          args: "-race;-short"
       - name: Test Integration
         uses: n8maninger/action-golang-test@v1
         with:

alerts/alerts.go

@@ -35,9 +35,9 @@ const (
 
 type (
 	Alerter interface {
+		Alerts(_ context.Context, opts AlertsOpts) (resp AlertsResponse, err error)
 		RegisterAlert(_ context.Context, a Alert) error
 		DismissAlerts(_ context.Context, ids ...types.Hash256) error
-		DismissAllAlerts(_ context.Context) error
 	}
 
 	// Severity indicates the severity of an alert.
@@ -66,11 +66,27 @@ type (
 	}
 
 	AlertsOpts struct {
-		Offset int
-		Limit  int
+		Offset   int
+		Limit    int
+		Severity Severity
+	}
+
+	AlertsResponse struct {
+		Alerts  []Alert `json:"alerts"`
+		HasMore bool    `json:"hasMore"`
+		Totals  struct {
+			Info     int `json:"info"`
+			Warning  int `json:"warning"`
+			Error    int `json:"error"`
+			Critical int `json:"critical"`
+		} `json:"totals"`
 	}
 )
 
+func (ar AlertsResponse) Total() int {
+	return ar.Totals.Info + ar.Totals.Warning + ar.Totals.Error + ar.Totals.Critical
+}
+
 // String implements the fmt.Stringer interface.
 func (s Severity) String() string {
 	switch s {
@@ -87,15 +103,8 @@ func (s Severity) String() string {
 	}
 }
 
-// MarshalJSON implements the json.Marshaler interface.
-func (s Severity) MarshalJSON() ([]byte, error) {
-	return []byte(fmt.Sprintf(`%q`, s.String())), nil
-}
-
-// UnmarshalJSON implements the json.Unmarshaler interface.
-func (s *Severity) UnmarshalJSON(b []byte) error {
-	status := strings.Trim(string(b), `"`)
-	switch status {
+func (s *Severity) LoadString(str string) error {
+	switch str {
 	case severityInfoStr:
 		*s = SeverityInfo
 	case severityWarningStr:
@@ -105,11 +114,21 @@ func (s *Severity) UnmarshalJSON(b []byte) error {
 	case severityCriticalStr:
 		*s = SeverityCritical
 	default:
-		return fmt.Errorf("unrecognized severity: %v", status)
+		return fmt.Errorf("unrecognized severity: %v", str)
 	}
 	return nil
 }
 
+// MarshalJSON implements the json.Marshaler interface.
+func (s Severity) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`%q`, s.String())), nil
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface.
+func (s *Severity) UnmarshalJSON(b []byte) error {
+	return s.LoadString(strings.Trim(string(b), `"`))
+}
+
 // RegisterAlert implements the Alerter interface.
 func (m *Manager) RegisterAlert(ctx context.Context, alert Alert) error {
 	if alert.ID == (types.Hash256{}) {
@@ -136,17 +155,6 @@ func (m *Manager) RegisterAlert(ctx context.Context, alert Alert) error {
 	})
 }
 
-// DismissAllAlerts implements the Alerter interface.
-func (m *Manager) DismissAllAlerts(ctx context.Context) error {
-	m.mu.Lock()
-	toDismiss := make([]types.Hash256, 0, len(m.alerts))
-	for alertID := range m.alerts {
-		toDismiss = append(toDismiss, alertID)
-	}
-	m.mu.Unlock()
-	return m.DismissAlerts(ctx, toDismiss...)
-}
-
 // DismissAlerts implements the Alerter interface.
 func (m *Manager) DismissAlerts(ctx context.Context, ids ...types.Hash256) error {
 	var dismissed []types.Hash256
@@ -175,19 +183,34 @@ func (m *Manager) DismissAlerts(ctx context.Context, ids ...types.Hash256) error
 	})
 }
 
-// Active returns the host's active alerts.
-func (m *Manager) Active(offset, limit int) []Alert {
+// Alerts returns the host's active alerts.
+func (m *Manager) Alerts(_ context.Context, opts AlertsOpts) (AlertsResponse, error) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
+	offset, limit := opts.Offset, opts.Limit
+	resp := AlertsResponse{}
+
 	if offset >= len(m.alerts) {
-		return nil
+		return resp, nil
 	} else if limit == -1 {
 		limit = len(m.alerts)
 	}
 
 	alerts := make([]Alert, 0, len(m.alerts))
 	for _, a := range m.alerts {
+		if a.Severity == SeverityInfo {
+			resp.Totals.Info++
+		} else if a.Severity == SeverityWarning {
+			resp.Totals.Warning++
+		} else if a.Severity == SeverityError {
+			resp.Totals.Error++
+		} else if a.Severity == SeverityCritical {
+			resp.Totals.Critical++
+		}
+		if opts.Severity != 0 && a.Severity != opts.Severity {
+			continue // filter by severity
+		}
 		alerts = append(alerts, a)
 	}
 	sort.Slice(alerts, func(i, j int) bool {
@@ -196,8 +219,10 @@ func (m *Manager) Active(offset, limit int) []Alert {
 	alerts = alerts[offset:]
 	if limit < len(alerts) {
 		alerts = alerts[:limit]
+		resp.HasMore = true
 	}
-	return alerts
+	resp.Alerts = alerts
+	return resp, nil
 }
 
 func (m *Manager) RegisterWebhookBroadcaster(b webhooks.Broadcaster) {
@@ -231,6 +256,11 @@ func WithOrigin(alerter Alerter, origin string) Alerter {
 	}
 }
 
+// Alerts implements the Alerter interface.
+func (a *originAlerter) Alerts(ctx context.Context, opts AlertsOpts) (resp AlertsResponse, err error) {
+	return a.alerter.Alerts(ctx, opts)
+}
+
 // RegisterAlert implements the Alerter interface.
 func (a *originAlerter) RegisterAlert(ctx context.Context, alert Alert) error {
 	if alert.Data == nil {
@@ -240,11 +270,6 @@ func (a *originAlerter) RegisterAlert(ctx context.Context, alert Alert) error {
 	return a.alerter.RegisterAlert(ctx, alert)
 }
 
-// DismissAllAlerts implements the Alerter interface.
-func (a *originAlerter) DismissAllAlerts(ctx context.Context) error {
-	return a.alerter.DismissAllAlerts(ctx)
-}
-
 // DismissAlerts implements the Alerter interface.
 func (a *originAlerter) DismissAlerts(ctx context.Context, ids ...types.Hash256) error {
 	return a.alerter.DismissAlerts(ctx, ids...)

api/object.go

@@ -54,7 +54,7 @@ type (
 	Object struct {
 		Metadata ObjectUserMetadata `json:"metadata,omitempty"`
 		ObjectMetadata
-		object.Object
+		*object.Object `json:"omitempty"`
 	}
 
 	// ObjectMetadata contains various metadata about an object.
@@ -212,13 +212,14 @@ type (
 	}
 
 	GetObjectOptions struct {
-		Prefix      string
-		Offset      int
-		Limit       int
-		IgnoreDelim bool
-		Marker      string
-		SortBy      string
-		SortDir     string
+		Prefix       string
+		Offset       int
+		Limit        int
+		IgnoreDelim  bool
+		Marker       string
+		OnlyMetadata bool
+		SortBy       string
+		SortDir      string
 	}
 
 	ListObjectOptions struct {
@@ -316,6 +317,9 @@ func (opts GetObjectOptions) Apply(values url.Values) {
 	if opts.Marker != "" {
 		values.Set("marker", opts.Marker)
 	}
+	if opts.OnlyMetadata {
+		values.Set("onlymetadata", "true")
+	}
 	if opts.SortBy != "" {
 		values.Set("sortBy", opts.SortBy)
 	}

autopilot/alerts.go

@@ -14,12 +14,13 @@ import (
 )
 
 var (
-	alertAccountRefillID = frand.Entropy256() // constant until restarted
-	alertLostSectorsID   = frand.Entropy256() // constant until restarted
-	alertLowBalanceID    = frand.Entropy256() // constant until restarted
-	alertMigrationID     = frand.Entropy256() // constant until restarted
-	alertPruningID       = frand.Entropy256() // constant until restarted
-	alertRenewalFailedID = frand.Entropy256() // constant until restarted
+	alertAccountRefillID = randomAlertID() // constant until restarted
+	alertChurnID         = randomAlertID() // constant until restarted
+	alertLostSectorsID   = randomAlertID() // constant until restarted
+	alertLowBalanceID    = randomAlertID() // constant until restarted
+	alertMigrationID     = randomAlertID() // constant until restarted
+	alertPruningID       = randomAlertID() // constant until restarted
+	alertRenewalFailedID = randomAlertID() // constant until restarted
 )
 
 func alertIDForAccount(alertID [32]byte, id rhpv3.Account) types.Hash256 {
@@ -54,6 +55,20 @@ func (ap *Autopilot) DismissAlert(ctx context.Context, ids ...types.Hash256) {
 	}
 }
 
+func (ap *Autopilot) HasAlert(ctx context.Context, id types.Hash256) bool {
+	ar, err := ap.alerts.Alerts(ctx, alerts.AlertsOpts{Offset: 0, Limit: -1})
+	if err != nil {
+		ap.logger.Errorf("failed to fetch alerts: %v", err)
+		return false
+	}
+	for _, alert := range ar.Alerts {
+		if alert.ID == id {
+			return true
+		}
+	}
+	return false
+}
+
 func newAccountLowBalanceAlert(address types.Address, balance, allowance types.Currency, bh, renewWindow, endHeight uint64) alerts.Alert {
 	severity := alerts.SeverityInfo
 	if bh+renewWindow/2 >= endHeight {
@@ -137,37 +152,6 @@ func newContractPruningFailedAlert(hk types.PublicKey, version string, fcid type
 	}
 }
 
-func newContractSetChangeAlert(name string, additions map[types.FileContractID]contractSetAddition, removals map[types.FileContractID]contractSetRemoval) alerts.Alert {
-	var hint string
-	if len(removals) > 0 {
-		hint = "A high churn rate can lead to a lot of unnecessary migrations, it might be necessary to tweak your configuration depending on the reason hosts are being discarded from the set."
-	}
-
-	removedReasons := make(map[string]string, len(removals))
-	for k, v := range removals {
-		removedReasons[k.String()] = v.Reason
-	}
-
-	return alerts.Alert{
-		ID:       randomAlertID(),
-		Severity: alerts.SeverityInfo,
-		Message:  "Contract set changed",
-		Data: map[string]any{
-			"name":          name,
-			"set_additions": additions,
-			"set_removals":  removals,
-			"hint":          hint,
-
-			// TODO: these fields can be removed on the next major release, they
-			// contain redundant information
-			"added":    len(additions),
-			"removed":  len(removals),
-			"removals": removedReasons,
-		},
-		Timestamp: time.Now(),
-	}
-}
-
 func newLostSectorsAlert(hk types.PublicKey, lostSectors uint64) alerts.Alert {
 	return alerts.Alert{
 		ID:       alertIDForHost(alertLostSectorsID, hk),

autopilot/churn.go

@@ -0,0 +1,68 @@
+package autopilot
+
+import (
+	"time"
+
+	"go.sia.tech/core/types"
+	"go.sia.tech/renterd/alerts"
+)
+
+type (
+	accumulatedChurn struct {
+		additions map[types.FileContractID]contractSetAdditions
+		removals  map[types.FileContractID]contractSetRemovals
+	}
+)
+
+func newAccumulatedChurn() *accumulatedChurn {
+	return &accumulatedChurn{
+		additions: make(map[types.FileContractID]contractSetAdditions),
+		removals:  make(map[types.FileContractID]contractSetRemovals),
+	}
+}
+
+func (c *accumulatedChurn) Alert(name string) alerts.Alert {
+	var hint string
+	if len(c.removals) > 0 {
+		hint = "A high churn rate can lead to a lot of unnecessary migrations, it might be necessary to tweak your configuration depending on the reason hosts are being discarded from the set."
+	}
+
+	return alerts.Alert{
+		ID:       alertChurnID,
+		Severity: alerts.SeverityInfo,
+		Message:  "Contract set changed",
+		Data: map[string]any{
+			"name":          name,
+			"set_additions": c.additions,
+			"set_removals":  c.removals,
+			"hint":          hint,
+		},
+		Timestamp: time.Now(),
+	}
+}
+
+func (c *accumulatedChurn) Apply(additions map[types.FileContractID]contractSetAdditions, removals map[types.FileContractID]contractSetRemovals) {
+	for fcid, a := range additions {
+		if _, exists := c.additions[fcid]; !exists {
+			c.additions[fcid] = a
+		} else {
+			additions := c.additions[fcid]
+			additions.Additions = append(additions.Additions, a.Additions...)
+			c.additions[fcid] = additions
+		}
+	}
+	for fcid, r := range removals {
+		if _, exists := c.removals[fcid]; !exists {
+			c.removals[fcid] = r
+		} else {
+			removals := c.removals[fcid]
+			removals.Removals = append(removals.Removals, r.Removals...)
+			c.removals[fcid] = removals
+		}
+	}
+}
+
+func (c *accumulatedChurn) Reset() {
+	c.additions = make(map[types.FileContractID]contractSetAdditions)
+	c.removals = make(map[types.FileContractID]contractSetRemovals)
+}