Merge branch 'dev' of github.com:SiaFoundation/renterd into pj/shutdo…

…wn-err

.github/workflows/test.yml

@@ -47,6 +47,16 @@ jobs:
         uses: n8maninger/action-golang-test@v1
         with:
           args: "-race;-short"
+      - name: Test Stores - MySQL
+        if: matrix.os == 'ubuntu-latest'
+        uses: n8maninger/action-golang-test@v1
+        env:
+          RENTERD_DB_URI: 127.0.0.1:3800
+          RENTERD_DB_USER: root
+          RENTERD_DB_PASSWORD: test
+        with:
+          package: "./stores"
+          args: "-race;-short"
       - name: Test Integration
         uses: n8maninger/action-golang-test@v1
         with:

alerts/alerts.go

@@ -35,6 +35,7 @@ const (
 
 type (
 	Alerter interface {
+		Alerts(_ context.Context, opts AlertsOpts) (resp AlertsResponse, err error)
 		RegisterAlert(_ context.Context, a Alert) error
 		DismissAlerts(_ context.Context, ids ...types.Hash256) error
 	}
@@ -63,8 +64,29 @@ type (
 		alerts             map[types.Hash256]Alert
 		webhookBroadcaster webhooks.Broadcaster
 	}
+
+	AlertsOpts struct {
+		Offset   int
+		Limit    int
+		Severity Severity
+	}
+
+	AlertsResponse struct {
+		Alerts  []Alert `json:"alerts"`
+		HasMore bool    `json:"hasMore"`
+		Totals  struct {
+			Info     int `json:"info"`
+			Warning  int `json:"warning"`
+			Error    int `json:"error"`
+			Critical int `json:"critical"`
+		} `json:"totals"`
+	}
 )
 
+func (ar AlertsResponse) Total() int {
+	return ar.Totals.Info + ar.Totals.Warning + ar.Totals.Error + ar.Totals.Critical
+}
+
 // String implements the fmt.Stringer interface.
 func (s Severity) String() string {
 	switch s {
@@ -81,15 +103,8 @@ func (s Severity) String() string {
 	}
 }
 
-// MarshalJSON implements the json.Marshaler interface.
-func (s Severity) MarshalJSON() ([]byte, error) {
-	return []byte(fmt.Sprintf(`%q`, s.String())), nil
-}
-
-// UnmarshalJSON implements the json.Unmarshaler interface.
-func (s *Severity) UnmarshalJSON(b []byte) error {
-	status := strings.Trim(string(b), `"`)
-	switch status {
+func (s *Severity) LoadString(str string) error {
+	switch str {
 	case severityInfoStr:
 		*s = SeverityInfo
 	case severityWarningStr:
@@ -99,11 +114,21 @@ func (s *Severity) UnmarshalJSON(b []byte) error {
 	case severityCriticalStr:
 		*s = SeverityCritical
 	default:
-		return fmt.Errorf("unrecognized severity: %v", status)
+		return fmt.Errorf("unrecognized severity: %v", str)
 	}
 	return nil
 }
 
+// MarshalJSON implements the json.Marshaler interface.
+func (s Severity) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`%q`, s.String())), nil
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface.
+func (s *Severity) UnmarshalJSON(b []byte) error {
+	return s.LoadString(strings.Trim(string(b), `"`))
+}
+
 // RegisterAlert implements the Alerter interface.
 func (m *Manager) RegisterAlert(ctx context.Context, alert Alert) error {
 	if alert.ID == (types.Hash256{}) {
@@ -158,19 +183,46 @@ func (m *Manager) DismissAlerts(ctx context.Context, ids ...types.Hash256) error
 	})
 }
 
-// Active returns the host's active alerts.
-func (m *Manager) Active() []Alert {
+// Alerts returns the host's active alerts.
+func (m *Manager) Alerts(_ context.Context, opts AlertsOpts) (AlertsResponse, error) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
+	offset, limit := opts.Offset, opts.Limit
+	resp := AlertsResponse{}
+
+	if offset >= len(m.alerts) {
+		return resp, nil
+	} else if limit == -1 {
+		limit = len(m.alerts)
+	}
+
 	alerts := make([]Alert, 0, len(m.alerts))
 	for _, a := range m.alerts {
+		if a.Severity == SeverityInfo {
+			resp.Totals.Info++
+		} else if a.Severity == SeverityWarning {
+			resp.Totals.Warning++
+		} else if a.Severity == SeverityError {
+			resp.Totals.Error++
+		} else if a.Severity == SeverityCritical {
+			resp.Totals.Critical++
+		}
+		if opts.Severity != 0 && a.Severity != opts.Severity {
+			continue // filter by severity
+		}
 		alerts = append(alerts, a)
 	}
 	sort.Slice(alerts, func(i, j int) bool {
 		return alerts[i].Timestamp.After(alerts[j].Timestamp)
 	})
-	return alerts
+	alerts = alerts[offset:]
+	if limit < len(alerts) {
+		alerts = alerts[:limit]
+		resp.HasMore = true
+	}
+	resp.Alerts = alerts
+	return resp, nil
 }
 
 func (m *Manager) RegisterWebhookBroadcaster(b webhooks.Broadcaster) {
@@ -204,6 +256,11 @@ func WithOrigin(alerter Alerter, origin string) Alerter {
 	}
 }
 
+// Alerts implements the Alerter interface.
+func (a *originAlerter) Alerts(ctx context.Context, opts AlertsOpts) (resp AlertsResponse, err error) {
+	return a.alerter.Alerts(ctx, opts)
+}
+
 // RegisterAlert implements the Alerter interface.
 func (a *originAlerter) RegisterAlert(ctx context.Context, alert Alert) error {
 	if alert.Data == nil {

api/multipart.go

@@ -46,9 +46,10 @@ type (
 	}
 
 	CreateMultipartOptions struct {
-		Key      object.EncryptionKey
-		MimeType string
-		Metadata ObjectUserMetadata
+		GenerateKey bool
+		Key         *object.EncryptionKey
+		MimeType    string
+		Metadata    ObjectUserMetadata
 	}
 )
 
@@ -81,11 +82,15 @@ type (
 	}
 
 	MultipartCreateRequest struct {
-		Bucket   string               `json:"bucket"`
-		Path     string               `json:"path"`
-		Key      object.EncryptionKey `json:"key"`
-		MimeType string               `json:"mimeType"`
-		Metadata ObjectUserMetadata   `json:"metadata"`
+		Bucket   string                `json:"bucket"`
+		Path     string                `json:"path"`
+		Key      *object.EncryptionKey `json:"key"`
+		MimeType string                `json:"mimeType"`
+		Metadata ObjectUserMetadata    `json:"metadata"`
+
+		// TODO: The next major version change should invert this to create a
+		// key by default
+		GenerateKey bool `json:"generateKey"`
 	}
 
 	MultipartCreateResponse struct {

api/object.go

@@ -54,7 +54,7 @@ type (
 	Object struct {
 		Metadata ObjectUserMetadata `json:"metadata,omitempty"`
 		ObjectMetadata
-		object.Object
+		*object.Object
 	}
 
 	// ObjectMetadata contains various metadata about an object.
@@ -119,6 +119,10 @@ type (
 		Mode   string `json:"mode"`
 	}
 
+	ObjectsStatsOpts struct {
+		Bucket string
+	}
+
 	// ObjectsStatsResponse is the response type for the /bus/stats/objects endpoint.
 	ObjectsStatsResponse struct {
 		NumObjects                 uint64  `json:"numObjects"`                 // number of objects
@@ -208,13 +212,14 @@ type (
 	}
 
 	GetObjectOptions struct {
-		Prefix      string
-		Offset      int
-		Limit       int
-		IgnoreDelim bool
-		Marker      string
-		SortBy      string
-		SortDir     string
+		Prefix       string
+		Offset       int
+		Limit        int
+		IgnoreDelim  bool
+		Marker       string
+		OnlyMetadata bool
+		SortBy       string
+		SortDir      string
 	}
 
 	ListObjectOptions struct {
@@ -231,20 +236,18 @@ type (
 
 	// UploadObjectOptions is the options type for the worker client.
 	UploadObjectOptions struct {
-		Offset                       int
-		MinShards                    int
-		TotalShards                  int
-		ContractSet                  string
-		DisablePreshardingEncryption bool
-		ContentLength                int64
-		MimeType                     string
-		Metadata                     ObjectUserMetadata
+		Offset        int
+		MinShards     int
+		TotalShards   int
+		ContractSet   string
+		ContentLength int64
+		MimeType      string
+		Metadata      ObjectUserMetadata
 	}
 
 	UploadMultipartUploadPartOptions struct {
-		DisablePreshardingEncryption bool
-		EncryptionOffset             int
-		ContentLength                int64
+		EncryptionOffset *int
+		ContentLength    int64
 	}
 )
 
@@ -264,9 +267,6 @@ func (opts UploadObjectOptions) ApplyValues(values url.Values) {
 	if opts.MimeType != "" {
 		values.Set("mimetype", opts.MimeType)
 	}
-	if opts.DisablePreshardingEncryption {
-		values.Set("disablepreshardingencryption", "true")
-	}
 }
 
 func (opts UploadObjectOptions) ApplyHeaders(h http.Header) {
@@ -276,11 +276,8 @@ func (opts UploadObjectOptions) ApplyHeaders(h http.Header) {
 }
 
 func (opts UploadMultipartUploadPartOptions) Apply(values url.Values) {
-	if opts.DisablePreshardingEncryption {
-		values.Set("disablepreshardingencryption", "true")
-	}
-	if !opts.DisablePreshardingEncryption || opts.EncryptionOffset != 0 {
-		values.Set("offset", fmt.Sprint(opts.EncryptionOffset))
+	if opts.EncryptionOffset != nil {
+		values.Set("offset", fmt.Sprint(*opts.EncryptionOffset))
 	}
 }
 
@@ -320,6 +317,9 @@ func (opts GetObjectOptions) Apply(values url.Values) {
 	if opts.Marker != "" {
 		values.Set("marker", opts.Marker)
 	}
+	if opts.OnlyMetadata {
+		values.Set("onlymetadata", "true")
+	}
 	if opts.SortBy != "" {
 		values.Set("sortBy", opts.SortBy)
 	}

api/setting.go

@@ -128,12 +128,12 @@ func (rs RedundancySettings) Redundancy() float64 {
 
 // SlabSize returns the size of a slab.
 func (rs RedundancySettings) SlabSize() uint64 {
-	return uint64(rs.MinShards) * rhpv2.SectorSize
+	return uint64(rs.TotalShards) * rhpv2.SectorSize
 }
 
-// SlabSizeWithRedundancy returns the size of a slab with redundancy.
-func (rs RedundancySettings) SlabSizeWithRedundancy() uint64 {
-	return uint64(rs.TotalShards) * rhpv2.SectorSize
+// SlabSizeNoRedundancy returns the size of a slab without added redundancy.
+func (rs RedundancySettings) SlabSizeNoRedundancy() uint64 {
+	return uint64(rs.MinShards) * rhpv2.SectorSize
 }
 
 // Validate returns an error if the redundancy settings are not considered

autopilot/alerts.go

@@ -14,12 +14,13 @@ import (
 )
 
 var (
-	alertAccountRefillID = frand.Entropy256() // constant until restarted
-	alertLostSectorsID   = frand.Entropy256() // constant until restarted
-	alertLowBalanceID    = frand.Entropy256() // constant until restarted
-	alertMigrationID     = frand.Entropy256() // constant until restarted
-	alertPruningID       = frand.Entropy256() // constant until restarted
-	alertRenewalFailedID = frand.Entropy256() // constant until restarted
+	alertAccountRefillID = randomAlertID() // constant until restarted
+	alertChurnID         = randomAlertID() // constant until restarted
+	alertLostSectorsID   = randomAlertID() // constant until restarted
+	alertLowBalanceID    = randomAlertID() // constant until restarted
+	alertMigrationID     = randomAlertID() // constant until restarted
+	alertPruningID       = randomAlertID() // constant until restarted
+	alertRenewalFailedID = randomAlertID() // constant until restarted
 )
 
 func alertIDForAccount(alertID [32]byte, id rhpv3.Account) types.Hash256 {
@@ -48,12 +49,26 @@ func (ap *Autopilot) RegisterAlert(ctx context.Context, a alerts.Alert) {
 	}
 }
 
-func (ap *Autopilot) DismissAlert(ctx context.Context, id types.Hash256) {
-	if err := ap.alerts.DismissAlerts(ctx, id); err != nil {
+func (ap *Autopilot) DismissAlert(ctx context.Context, ids ...types.Hash256) {
+	if err := ap.alerts.DismissAlerts(ctx, ids...); err != nil {
 		ap.logger.Errorf("failed to dismiss alert: %v", err)
 	}
 }
 
+func (ap *Autopilot) HasAlert(ctx context.Context, id types.Hash256) bool {
+	ar, err := ap.alerts.Alerts(ctx, alerts.AlertsOpts{Offset: 0, Limit: -1})
+	if err != nil {
+		ap.logger.Errorf("failed to fetch alerts: %v", err)
+		return false
+	}
+	for _, alert := range ar.Alerts {
+		if alert.ID == id {
+			return true
+		}
+	}
+	return false
+}
+
 func newAccountLowBalanceAlert(address types.Address, balance, allowance types.Currency, bh, renewWindow, endHeight uint64) alerts.Alert {
 	severity := alerts.SeverityInfo
 	if bh+renewWindow/2 >= endHeight {
@@ -137,27 +152,6 @@ func newContractPruningFailedAlert(hk types.PublicKey, version string, fcid type
 	}
 }
 
-func newContractSetChangeAlert(name string, added, removed int, removedReasons map[string]string) alerts.Alert {
-	var hint string
-	if removed > 0 {
-		hint = "A high churn rate can lead to a lot of unnecessary migrations, it might be necessary to tweak your configuration depending on the reason hosts are being discarded from the set."
-	}
-
-	return alerts.Alert{
-		ID:       randomAlertID(),
-		Severity: alerts.SeverityInfo,
-		Message:  "Contract set changed",
-		Data: map[string]any{
-			"name":     name,
-			"added":    added,
-			"removed":  removed,
-			"removals": removedReasons,
-			"hint":     hint,
-		},
-		Timestamp: time.Now(),
-	}
-}
-
 func newLostSectorsAlert(hk types.PublicKey, lostSectors uint64) alerts.Alert {
 	return alerts.Alert{
 		ID:       alertIDForHost(alertLostSectorsID, hk),