Merge branch 'pj/mock-bus' into pj/fix-packed-slab-uploads

.github/workflows/test.yml

@@ -60,7 +60,7 @@ jobs:
       - name: Test Integration
         uses: n8maninger/action-golang-test@v1
         with:
-          package: "./internal/testing/..."
+          package: "./internal/test/e2e/..."
           args: "-failfast;-race;-tags=testing;-timeout=30m"
       - name: Test Integration - MySQL
         if: matrix.os == 'ubuntu-latest'
@@ -70,7 +70,7 @@ jobs:
           RENTERD_DB_USER: root
           RENTERD_DB_PASSWORD: test
         with:
-          package: "./internal/testing/..."
+          package: "./internal/test/e2e/..."
           args: "-failfast;-race;-tags=testing;-timeout=30m"
       - name: Build
         run: go build -o bin/ ./cmd/renterd

alerts/alerts.go

@@ -66,17 +66,27 @@ type (
 	}
 
 	AlertsOpts struct {
-		Offset int
-		Limit  int
+		Offset   int
+		Limit    int
+		Severity Severity
 	}
 
 	AlertsResponse struct {
 		Alerts  []Alert `json:"alerts"`
 		HasMore bool    `json:"hasMore"`
-		Total   int     `json:"total"`
+		Totals  struct {
+			Info     int `json:"info"`
+			Warning  int `json:"warning"`
+			Error    int `json:"error"`
+			Critical int `json:"critical"`
+		} `json:"totals"`
 	}
 )
 
+func (ar AlertsResponse) Total() int {
+	return ar.Totals.Info + ar.Totals.Warning + ar.Totals.Error + ar.Totals.Critical
+}
+
 // String implements the fmt.Stringer interface.
 func (s Severity) String() string {
 	switch s {
@@ -93,15 +103,8 @@ func (s Severity) String() string {
 	}
 }
 
-// MarshalJSON implements the json.Marshaler interface.
-func (s Severity) MarshalJSON() ([]byte, error) {
-	return []byte(fmt.Sprintf(`%q`, s.String())), nil
-}
-
-// UnmarshalJSON implements the json.Unmarshaler interface.
-func (s *Severity) UnmarshalJSON(b []byte) error {
-	status := strings.Trim(string(b), `"`)
-	switch status {
+func (s *Severity) LoadString(str string) error {
+	switch str {
 	case severityInfoStr:
 		*s = SeverityInfo
 	case severityWarningStr:
@@ -111,11 +114,21 @@ func (s *Severity) UnmarshalJSON(b []byte) error {
 	case severityCriticalStr:
 		*s = SeverityCritical
 	default:
-		return fmt.Errorf("unrecognized severity: %v", status)
+		return fmt.Errorf("unrecognized severity: %v", str)
 	}
 	return nil
 }
 
+// MarshalJSON implements the json.Marshaler interface.
+func (s Severity) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`%q`, s.String())), nil
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface.
+func (s *Severity) UnmarshalJSON(b []byte) error {
+	return s.LoadString(strings.Trim(string(b), `"`))
+}
+
 // RegisterAlert implements the Alerter interface.
 func (m *Manager) RegisterAlert(ctx context.Context, alert Alert) error {
 	if alert.ID == (types.Hash256{}) {
@@ -176,9 +189,7 @@ func (m *Manager) Alerts(_ context.Context, opts AlertsOpts) (AlertsResponse, er
 	defer m.mu.Unlock()
 
 	offset, limit := opts.Offset, opts.Limit
-	resp := AlertsResponse{
-		Total: len(m.alerts),
-	}
+	resp := AlertsResponse{}
 
 	if offset >= len(m.alerts) {
 		return resp, nil
@@ -188,6 +199,18 @@ func (m *Manager) Alerts(_ context.Context, opts AlertsOpts) (AlertsResponse, er
 
 	alerts := make([]Alert, 0, len(m.alerts))
 	for _, a := range m.alerts {
+		if a.Severity == SeverityInfo {
+			resp.Totals.Info++
+		} else if a.Severity == SeverityWarning {
+			resp.Totals.Warning++
+		} else if a.Severity == SeverityError {
+			resp.Totals.Error++
+		} else if a.Severity == SeverityCritical {
+			resp.Totals.Critical++
+		}
+		if opts.Severity != 0 && a.Severity != opts.Severity {
+			continue // filter by severity
+		}
 		alerts = append(alerts, a)
 	}
 	sort.Slice(alerts, func(i, j int) bool {

api/object.go

@@ -54,7 +54,7 @@ type (
 	Object struct {
 		Metadata ObjectUserMetadata `json:"metadata,omitempty"`
 		ObjectMetadata
-		*object.Object `json:"omitempty"`
+		*object.Object
 	}
 
 	// ObjectMetadata contains various metadata about an object.

bus/bus.go

@@ -1741,15 +1741,22 @@ func (b *bus) handleGETAlerts(jc jape.Context) {
 		return
 	}
 	offset, limit := 0, -1
+	var severity alerts.Severity
 	if jc.DecodeForm("offset", &offset) != nil {
 		return
 	} else if jc.DecodeForm("limit", &limit) != nil {
 		return
 	} else if offset < 0 {
 		jc.Error(errors.New("offset must be non-negative"), http.StatusBadRequest)
 		return
+	} else if jc.DecodeForm("severity", &severity) != nil {
+		return
 	}
-	ar, err := b.alertMgr.Alerts(jc.Request.Context(), alerts.AlertsOpts{Offset: offset, Limit: limit})
+	ar, err := b.alertMgr.Alerts(jc.Request.Context(), alerts.AlertsOpts{
+		Offset:   offset,
+		Limit:    limit,
+		Severity: severity,
+	})
 	if jc.Check("failed to fetch alerts", err) != nil {
 		return
 	}

bus/client/alerts.go

@@ -16,6 +16,9 @@ func (c *Client) Alerts(ctx context.Context, opts alerts.AlertsOpts) (resp alert
 	if opts.Limit != 0 {
 		values.Set("limit", fmt.Sprint(opts.Limit))
 	}
+	if opts.Severity != 0 {
+		values.Set("severity", opts.Severity.String())
+	}
 	err = c.c.WithContext(ctx).GET("/alerts?"+values.Encode(), &resp)
 	return
 }

internal/test/config.go

@@ -0,0 +1,64 @@
+package test
+
+import (
+	"time"
+
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	rhpv2 "go.sia.tech/core/rhp/v2"
+	"go.sia.tech/core/types"
+	"go.sia.tech/renterd/api"
+)
+
+var (
+	// AutopilotConfig is the autopilot used for testing unless a different
+	// one is explicitly set.
+	AutopilotConfig = api.AutopilotConfig{
+		Contracts: api.ContractsConfig{
+			Allowance:   types.Siacoins(1).Mul64(1e3),
+			Amount:      3,
+			Period:      144,
+			RenewWindow: 72,
+
+			Download: rhpv2.SectorSize * 500,
+			Upload:   rhpv2.SectorSize * 500,
+			Storage:  rhpv2.SectorSize * 5e3,
+
+			Set:   ContractSet,
+			Prune: false,
+		},
+		Hosts: api.HostsConfig{
+			MaxDowntimeHours:      10,
+			MinRecentScanFailures: 10,
+			AllowRedundantIPs:     true, // allow for integration tests by default
+		},
+	}
+
+	ContractSet         = "testset"
+	ContractSetSettings = api.ContractSetSetting{
+		Default: ContractSet,
+	}
+
+	GougingSettings = api.GougingSettings{
+		MinMaxCollateral: types.Siacoins(10),                   // at least up to 10 SC per contract
+		MaxRPCPrice:      types.Siacoins(1).Div64(1000),        // 1mS per RPC
+		MaxContractPrice: types.Siacoins(10),                   // 10 SC per contract
+		MaxDownloadPrice: types.Siacoins(1).Mul64(1000),        // 1000 SC per 1 TiB
+		MaxUploadPrice:   types.Siacoins(1).Mul64(1000),        // 1000 SC per 1 TiB
+		MaxStoragePrice:  types.Siacoins(1000).Div64(144 * 30), // 1000 SC per month
+
+		HostBlockHeightLeeway: 240, // amount of leeway given to host block height
+
+		MinPriceTableValidity:         10 * time.Second,  // minimum value for price table validity
+		MinAccountExpiry:              time.Hour,         // minimum value for account expiry
+		MinMaxEphemeralAccountBalance: types.Siacoins(1), // 1SC
+	}
+
+	RedundancySettings = api.RedundancySettings{
+		MinShards:   2,
+		TotalShards: 3,
+	}
+
+	S3AccessKeyID     = "TESTINGYNHUWCPKOPSYQ"
+	S3SecretAccessKey = "Rh30BNyj+qNI4ftYRteoZbHJ3X4Ln71QtZkRXzJ9"
+	S3Credentials     = credentials.NewStaticV4(S3AccessKeyID, S3SecretAccessKey, "")
+)

internal/testing/blocklist_test.go → internal/test/e2e/blocklist_test.go

@@ -1,4 +1,4 @@
-package testing
+package e2e
 
 import (
 	"context"
@@ -8,6 +8,7 @@ import (
 
 	"go.sia.tech/core/types"
 	"go.sia.tech/renterd/api"
+	"go.sia.tech/renterd/internal/test"
 )
 
 func TestBlocklist(t *testing.T) {
@@ -26,7 +27,7 @@ func TestBlocklist(t *testing.T) {
 	tt := cluster.tt
 
 	// fetch contracts
-	contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: testAutopilotConfig.Contracts.Set})
+	contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: test.AutopilotConfig.Contracts.Set})
 	tt.OK(err)
 	if len(contracts) != 3 {
 		t.Fatalf("unexpected number of contracts, %v != 3", len(contracts))
@@ -40,7 +41,7 @@ func TestBlocklist(t *testing.T) {
 
 	// assert h3 is no longer in the contract set
 	tt.Retry(5, time.Second, func() error {
-		contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: testAutopilotConfig.Contracts.Set})
+		contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: test.AutopilotConfig.Contracts.Set})
 		tt.OK(err)
 		if len(contracts) != 2 {
 			return fmt.Errorf("unexpected number of contracts, %v != 2", len(contracts))
@@ -60,7 +61,7 @@ func TestBlocklist(t *testing.T) {
 
 	// assert h1 is no longer in the contract set
 	tt.Retry(5, time.Second, func() error {
-		contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: testAutopilotConfig.Contracts.Set})
+		contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: test.AutopilotConfig.Contracts.Set})
 		tt.OK(err)
 		if len(contracts) != 1 {
 			return fmt.Errorf("unexpected number of contracts, %v != 1", len(contracts))
@@ -77,7 +78,7 @@ func TestBlocklist(t *testing.T) {
 	tt.OK(b.UpdateHostAllowlist(ctx, nil, []types.PublicKey{hk1, hk2}, false))
 	tt.OK(b.UpdateHostBlocklist(ctx, nil, []string{h1.NetAddress}, false))
 	tt.Retry(5, time.Second, func() error {
-		contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: testAutopilotConfig.Contracts.Set})
+		contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: test.AutopilotConfig.Contracts.Set})
 		tt.OK(err)
 		if len(contracts) != 3 {
 			return fmt.Errorf("unexpected number of contracts, %v != 3", len(contracts))