From 0ef1757537003c419df3909b925d9f1d76f064a9 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 27 Feb 2024 10:55:58 +0100 Subject: [PATCH 01/12] bus: filter by alert severity --- alerts/alerts.go | 55 +++++++++++++++++++++----------- bus/bus.go | 9 +++++- bus/client/alerts.go | 3 ++ internal/testing/cluster_test.go | 38 ++++++++++++++++++++++ 4 files changed, 86 insertions(+), 19 deletions(-) diff --git a/alerts/alerts.go b/alerts/alerts.go index f11004dbe..a14d460b6 100644 --- a/alerts/alerts.go +++ b/alerts/alerts.go @@ -66,14 +66,19 @@ type ( } AlertsOpts struct { - Offset int - Limit int + Offset int + Limit int + Severity Severity } AlertsResponse struct { - Alerts []Alert `json:"alerts"` - HasMore bool `json:"hasMore"` - Total int `json:"total"` + Alerts []Alert `json:"alerts"` + HasMore bool `json:"hasMore"` + Total int `json:"total"` + TotalInfo int `json:"totalInfo"` + TotalWarning int `json:"totalWarning"` + TotalError int `json:"totalError"` + TotalCritical int `json:"totalCritical"` } ) @@ -93,15 +98,8 @@ func (s Severity) String() string { } } -// MarshalJSON implements the json.Marshaler interface. -func (s Severity) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf(`%q`, s.String())), nil -} - -// UnmarshalJSON implements the json.Unmarshaler interface. -func (s *Severity) UnmarshalJSON(b []byte) error { - status := strings.Trim(string(b), `"`) - switch status { +func (s *Severity) LoadString(str string) error { + switch str { case severityInfoStr: *s = SeverityInfo case severityWarningStr: @@ -111,11 +109,21 @@ func (s *Severity) UnmarshalJSON(b []byte) error { case severityCriticalStr: *s = SeverityCritical default: - return fmt.Errorf("unrecognized severity: %v", status) + return fmt.Errorf("unrecognized severity: %v", str) } return nil } +// MarshalJSON implements the json.Marshaler interface. +func (s Severity) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`%q`, s.String())), nil +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (s *Severity) UnmarshalJSON(b []byte) error { + return s.LoadString(strings.Trim(string(b), `"`)) +} + // RegisterAlert implements the Alerter interface. func (m *Manager) RegisterAlert(ctx context.Context, alert Alert) error { if alert.ID == (types.Hash256{}) { @@ -176,9 +184,7 @@ func (m *Manager) Alerts(_ context.Context, opts AlertsOpts) (AlertsResponse, er defer m.mu.Unlock() offset, limit := opts.Offset, opts.Limit - resp := AlertsResponse{ - Total: len(m.alerts), - } + resp := AlertsResponse{} if offset >= len(m.alerts) { return resp, nil @@ -188,6 +194,19 @@ func (m *Manager) Alerts(_ context.Context, opts AlertsOpts) (AlertsResponse, er alerts := make([]Alert, 0, len(m.alerts)) for _, a := range m.alerts { + resp.Total++ + if a.Severity == SeverityInfo { + resp.TotalInfo++ + } else if a.Severity == SeverityWarning { + resp.TotalWarning++ + } else if a.Severity == SeverityError { + resp.TotalError++ + } else if a.Severity == SeverityCritical { + resp.TotalCritical++ + } + if opts.Severity != 0 && a.Severity != opts.Severity { + continue // filter by severity + } alerts = append(alerts, a) } sort.Slice(alerts, func(i, j int) bool { diff --git a/bus/bus.go b/bus/bus.go index e7e6ddaac..4106bc231 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -1739,6 +1739,7 @@ func (b *bus) handleGETAlerts(jc jape.Context) { return } offset, limit := 0, -1 + var severity alerts.Severity if jc.DecodeForm("offset", &offset) != nil { return } else if jc.DecodeForm("limit", &limit) != nil { @@ -1746,8 +1747,14 @@ func (b *bus) handleGETAlerts(jc jape.Context) { } else if offset < 0 { jc.Error(errors.New("offset must be non-negative"), http.StatusBadRequest) return + } else if jc.DecodeForm("severity", &severity) != nil { + return } - ar, err := b.alertMgr.Alerts(jc.Request.Context(), alerts.AlertsOpts{Offset: offset, Limit: limit}) + ar, err := b.alertMgr.Alerts(jc.Request.Context(), alerts.AlertsOpts{ + Offset: offset, + Limit: limit, + Severity: severity, + }) if jc.Check("failed to fetch alerts", err) != nil { return } diff --git a/bus/client/alerts.go b/bus/client/alerts.go index 7eceaeaed..28c3b9a84 100644 --- a/bus/client/alerts.go +++ b/bus/client/alerts.go @@ -16,6 +16,9 @@ func (c *Client) Alerts(ctx context.Context, opts alerts.AlertsOpts) (resp alert if opts.Limit != 0 { values.Set("limit", fmt.Sprint(opts.Limit)) } + if opts.Severity != 0 { + values.Set("severity", opts.Severity.String()) + } err = c.c.WithContext(ctx).GET("/alerts?"+values.Encode(), &resp) return } diff --git a/internal/testing/cluster_test.go b/internal/testing/cluster_test.go index f30a0906a..69b318f66 100644 --- a/internal/testing/cluster_test.go +++ b/internal/testing/cluster_test.go @@ -1974,6 +1974,44 @@ func TestAlerts(t *testing.T) { if len(foundAlerts) != 1 || foundAlerts[0].ID != alert2.ID { t.Fatal("wrong alert") } + + // register more alerts + for severity := alerts.SeverityInfo; severity <= alerts.SeverityCritical; severity++ { + for j := 0; j < 3*int(severity); j++ { + tt.OK(b.RegisterAlert(context.Background(), alerts.Alert{ + ID: frand.Entropy256(), + Severity: severity, + Message: "test", + Data: map[string]interface{}{ + "origin": "test", + }, + Timestamp: time.Now(), + })) + } + } + for severity := alerts.SeverityInfo; severity <= alerts.SeverityCritical; severity++ { + ar, err = b.Alerts(context.Background(), alerts.AlertsOpts{Severity: severity}) + tt.OK(err) + if ar.Total != 32 { + t.Fatal("expected 32 alerts", ar.Total) + } else if ar.TotalInfo != 3 { + t.Fatal("expected 3 info alerts", ar.TotalInfo) + } else if ar.TotalWarning != 6 { + t.Fatal("expected 6 warning alerts", ar.TotalWarning) + } else if ar.TotalError != 9 { + t.Fatal("expected 9 error alerts", ar.TotalError) + } else if ar.TotalCritical != 14 { + t.Fatal("expected 14 critical alerts", ar.TotalCritical) + } else if severity == alerts.SeverityInfo && len(ar.Alerts) != ar.TotalInfo { + t.Fatalf("expected %v info alerts, got %v", ar.TotalInfo, len(ar.Alerts)) + } else if severity == alerts.SeverityWarning && len(ar.Alerts) != ar.TotalWarning { + t.Fatalf("expected %v warning alerts, got %v", ar.TotalWarning, len(ar.Alerts)) + } else if severity == alerts.SeverityError && len(ar.Alerts) != ar.TotalError { + t.Fatalf("expected %v error alerts, got %v", ar.TotalError, len(ar.Alerts)) + } else if severity == alerts.SeverityCritical && len(ar.Alerts) != ar.TotalCritical { + t.Fatalf("expected %v critical alerts, got %v", ar.TotalCritical, len(ar.Alerts)) + } + } } func TestMultipartUploads(t *testing.T) { From 7c5b3c91efab2acf8f4b90588dace1240d00c0f7 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 27 Feb 2024 13:56:29 +0100 Subject: [PATCH 02/12] bus: breakdown totals --- alerts/alerts.go | 28 ++++++++++++++----------- internal/testing/cluster_test.go | 36 ++++++++++++++++---------------- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/alerts/alerts.go b/alerts/alerts.go index a14d460b6..1ebebbc80 100644 --- a/alerts/alerts.go +++ b/alerts/alerts.go @@ -72,16 +72,21 @@ type ( } AlertsResponse struct { - Alerts []Alert `json:"alerts"` - HasMore bool `json:"hasMore"` - Total int `json:"total"` - TotalInfo int `json:"totalInfo"` - TotalWarning int `json:"totalWarning"` - TotalError int `json:"totalError"` - TotalCritical int `json:"totalCritical"` + Alerts []Alert `json:"alerts"` + HasMore bool `json:"hasMore"` + Totals struct { + Info int `json:"info"` + Warning int `json:"warning"` + Error int `json:"error"` + Critical int `json:"critical"` + } `json:"total"` } ) +func (ar AlertsResponse) Total() int { + return ar.Totals.Info + ar.Totals.Warning + ar.Totals.Error + ar.Totals.Critical +} + // String implements the fmt.Stringer interface. func (s Severity) String() string { switch s { @@ -194,15 +199,14 @@ func (m *Manager) Alerts(_ context.Context, opts AlertsOpts) (AlertsResponse, er alerts := make([]Alert, 0, len(m.alerts)) for _, a := range m.alerts { - resp.Total++ if a.Severity == SeverityInfo { - resp.TotalInfo++ + resp.Totals.Info++ } else if a.Severity == SeverityWarning { - resp.TotalWarning++ + resp.Totals.Warning++ } else if a.Severity == SeverityError { - resp.TotalError++ + resp.Totals.Error++ } else if a.Severity == SeverityCritical { - resp.TotalCritical++ + resp.Totals.Critical++ } if opts.Severity != 0 && a.Severity != opts.Severity { continue // filter by severity diff --git a/internal/testing/cluster_test.go b/internal/testing/cluster_test.go index 69b318f66..6b5f88769 100644 --- a/internal/testing/cluster_test.go +++ b/internal/testing/cluster_test.go @@ -1992,24 +1992,24 @@ func TestAlerts(t *testing.T) { for severity := alerts.SeverityInfo; severity <= alerts.SeverityCritical; severity++ { ar, err = b.Alerts(context.Background(), alerts.AlertsOpts{Severity: severity}) tt.OK(err) - if ar.Total != 32 { - t.Fatal("expected 32 alerts", ar.Total) - } else if ar.TotalInfo != 3 { - t.Fatal("expected 3 info alerts", ar.TotalInfo) - } else if ar.TotalWarning != 6 { - t.Fatal("expected 6 warning alerts", ar.TotalWarning) - } else if ar.TotalError != 9 { - t.Fatal("expected 9 error alerts", ar.TotalError) - } else if ar.TotalCritical != 14 { - t.Fatal("expected 14 critical alerts", ar.TotalCritical) - } else if severity == alerts.SeverityInfo && len(ar.Alerts) != ar.TotalInfo { - t.Fatalf("expected %v info alerts, got %v", ar.TotalInfo, len(ar.Alerts)) - } else if severity == alerts.SeverityWarning && len(ar.Alerts) != ar.TotalWarning { - t.Fatalf("expected %v warning alerts, got %v", ar.TotalWarning, len(ar.Alerts)) - } else if severity == alerts.SeverityError && len(ar.Alerts) != ar.TotalError { - t.Fatalf("expected %v error alerts, got %v", ar.TotalError, len(ar.Alerts)) - } else if severity == alerts.SeverityCritical && len(ar.Alerts) != ar.TotalCritical { - t.Fatalf("expected %v critical alerts, got %v", ar.TotalCritical, len(ar.Alerts)) + if ar.Total() != 32 { + t.Fatal("expected 32 alerts", ar.Total()) + } else if ar.Totals.Info != 3 { + t.Fatal("expected 3 info alerts", ar.Totals.Info) + } else if ar.Totals.Warning != 6 { + t.Fatal("expected 6 warning alerts", ar.Totals.Warning) + } else if ar.Totals.Error != 9 { + t.Fatal("expected 9 error alerts", ar.Totals.Error) + } else if ar.Totals.Critical != 14 { + t.Fatal("expected 14 critical alerts", ar.Totals.Critical) + } else if severity == alerts.SeverityInfo && len(ar.Alerts) != ar.Totals.Info { + t.Fatalf("expected %v info alerts, got %v", ar.Totals.Info, len(ar.Alerts)) + } else if severity == alerts.SeverityWarning && len(ar.Alerts) != ar.Totals.Warning { + t.Fatalf("expected %v warning alerts, got %v", ar.Totals.Warning, len(ar.Alerts)) + } else if severity == alerts.SeverityError && len(ar.Alerts) != ar.Totals.Error { + t.Fatalf("expected %v error alerts, got %v", ar.Totals.Error, len(ar.Alerts)) + } else if severity == alerts.SeverityCritical && len(ar.Alerts) != ar.Totals.Critical { + t.Fatalf("expected %v critical alerts, got %v", ar.Totals.Critical, len(ar.Alerts)) } } } From 6db1e38cf0f415f330c6ba54a24682af69d1dc16 Mon Sep 17 00:00:00 2001 From: Christopher Schinnerl Date: Tue, 27 Feb 2024 13:58:47 +0100 Subject: [PATCH 03/12] Update alerts/alerts.go Co-authored-by: Peter-Jan Brone --- alerts/alerts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerts/alerts.go b/alerts/alerts.go index 1ebebbc80..6b009360d 100644 --- a/alerts/alerts.go +++ b/alerts/alerts.go @@ -79,7 +79,7 @@ type ( Warning int `json:"warning"` Error int `json:"error"` Critical int `json:"critical"` - } `json:"total"` + } `json:"totals"` } ) From 0fceed12cbba2304bc17f75c287e9272aaa8c08e Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 27 Feb 2024 14:54:11 +0100 Subject: [PATCH 04/12] api: fix Object response type --- api/object.go | 2 +- object/object.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/object.go b/api/object.go index 35df9b636..cef672a97 100644 --- a/api/object.go +++ b/api/object.go @@ -54,7 +54,7 @@ type ( Object struct { Metadata ObjectUserMetadata `json:"metadata,omitempty"` ObjectMetadata - *object.Object `json:"omitempty"` + *object.Object } // ObjectMetadata contains various metadata about an object. diff --git a/object/object.go b/object/object.go index 2331f6251..a545a4935 100644 --- a/object/object.go +++ b/object/object.go @@ -115,8 +115,8 @@ func GenerateEncryptionKey() EncryptionKey { // An Object is a unit of data that has been stored on a host. type Object struct { - Key EncryptionKey `json:"key"` - Slabs []SlabSlice `json:"slabs"` + Key EncryptionKey `json:"key,omitempty"` + Slabs []SlabSlice `json:"slabs,omitempty"` } // NewObject returns a new Object with a random key. From f29534aef0b996e95fd36cbd1b914f72a6c3fa04 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 27 Feb 2024 15:17:50 +0100 Subject: [PATCH 05/12] object: add docstring to Object type --- object/object.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/object/object.go b/object/object.go index a545a4935..965ebce2a 100644 --- a/object/object.go +++ b/object/object.go @@ -114,6 +114,9 @@ func GenerateEncryptionKey() EncryptionKey { } // An Object is a unit of data that has been stored on a host. +// NOTE: Object is embedded in the API's Object type, so all fields should be +// tagged omitempty to make sure responses where no object is returned remain +// clean. type Object struct { Key EncryptionKey `json:"key,omitempty"` Slabs []SlabSlice `json:"slabs,omitempty"` From a704e9fbae284b892651e047152890819d9856eb Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 27 Feb 2024 15:27:34 +0100 Subject: [PATCH 06/12] testing: add testutils package --- internal/testing/cluster.go | 67 ++------------------ internal/testing/cluster_test.go | 4 +- testutils/tt.go | 101 +++++++++++++++++++++++++++++++ worker/host_test.go | 9 +-- worker/worker_test.go | 19 +++--- 5 files changed, 121 insertions(+), 79 deletions(-) create mode 100644 testutils/tt.go diff --git a/internal/testing/cluster.go b/internal/testing/cluster.go index d55539cd7..d36f143be 100644 --- a/internal/testing/cluster.go +++ b/internal/testing/cluster.go @@ -9,7 +9,6 @@ import ( "net/http" "os" "path/filepath" - "strings" "sync" "testing" "time" @@ -27,6 +26,7 @@ import ( "go.sia.tech/renterd/internal/node" "go.sia.tech/renterd/s3" "go.sia.tech/renterd/stores" + "go.sia.tech/renterd/testutils" "go.uber.org/zap" "go.uber.org/zap/zapcore" "gorm.io/gorm" @@ -98,47 +98,6 @@ var ( testS3Credentials = credentials.NewStaticV4(testS3AccessKeyID, testS3SecretAccessKey, "") ) -type TT struct { - *testing.T -} - -func (t TT) AssertContains(err error, target string) { - t.Helper() - if err == nil || !strings.Contains(err.Error(), target) { - t.Fatalf("err: %v != target: %v", err, target) - } -} - -func (t TT) AssertIs(err, target error) { - t.Helper() - t.AssertContains(err, target.Error()) -} - -func (t TT) OK(err error) { - t.Helper() - if err != nil { - t.Fatal(err) - } -} - -func (t TT) OKAll(vs ...interface{}) { - t.Helper() - for _, v := range vs { - if err, ok := v.(error); ok && err != nil { - t.Fatal(err) - } - } -} - -func (t TT) FailAll(vs ...interface{}) { - t.Helper() - for _, v := range vs { - if err, ok := v.(error); ok && err == nil { - t.Fatal("should've failed") - } - } -} - // TestCluster is a helper type that allows for easily creating a number of // nodes connected to each other and ready for testing. type TestCluster struct { @@ -161,7 +120,7 @@ type TestCluster struct { dbName string dir string logger *zap.Logger - tt *TT + tt testutils.TT wk types.PrivateKey wg sync.WaitGroup } @@ -203,33 +162,17 @@ func randomPassword() string { return hex.EncodeToString(frand.Bytes(32)) } -// Retry will call 'fn' 'tries' times, waiting 'durationBetweenAttempts' -// between each attempt, returning 'nil' the first time that 'fn' returns nil. -// If 'nil' is never returned, then the final error returned by 'fn' is -// returned. -func (tt *TT) Retry(tries int, durationBetweenAttempts time.Duration, fn func() error) { - tt.Helper() - for i := 1; i < tries; i++ { - err := fn() - if err == nil { - return - } - time.Sleep(durationBetweenAttempts) - } - tt.OK(fn()) -} - // Reboot simulates a reboot of the cluster by calling Shutdown and creating a // new cluster using the same settings as the previous one. // NOTE: Simulating a reboot means that the hosts stay active and are not // restarted. -func (c *TestCluster) Reboot(ctx context.Context) *TestCluster { +func (c *TestCluster) Reboot(t *testing.T) *TestCluster { c.tt.Helper() hosts := c.hosts c.hosts = nil c.Shutdown() - newCluster := newTestCluster(c.tt.T, testClusterOptions{ + newCluster := newTestCluster(t, testClusterOptions{ dir: c.dir, dbName: c.dbName, logger: c.logger, @@ -302,7 +245,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { if testing.Short() { t.SkipNow() } - tt := &TT{t} + tt := testutils.New(t) // Ensure we don't hang ctx, cancel := context.WithTimeout(context.Background(), time.Minute) diff --git a/internal/testing/cluster_test.go b/internal/testing/cluster_test.go index f30a0906a..d7c458d8b 100644 --- a/internal/testing/cluster_test.go +++ b/internal/testing/cluster_test.go @@ -1052,7 +1052,7 @@ func TestEphemeralAccounts(t *testing.T) { } // Reboot cluster. - cluster2 := cluster.Reboot(context.Background()) + cluster2 := cluster.Reboot(t) defer cluster2.Shutdown() // Check that accounts were loaded from the bus. @@ -1246,7 +1246,7 @@ func TestEphemeralAccountSync(t *testing.T) { } // Restart cluster to have worker fetch the account from the bus again. - cluster2 := cluster.Reboot(context.Background()) + cluster2 := cluster.Reboot(t) defer cluster2.Shutdown() // Account should need a sync. diff --git a/testutils/tt.go b/testutils/tt.go new file mode 100644 index 000000000..9345a2fd7 --- /dev/null +++ b/testutils/tt.go @@ -0,0 +1,101 @@ +package testutils + +import ( + "strings" + "time" +) + +type ( + TT interface { + TestingCommon + + AssertContains(err error, target string) + AssertIs(err, target error) + FailAll(vs ...interface{}) + OK(err error) + OKAll(vs ...interface{}) + + // Retry will call 'fn' 'tries' times, waiting 'durationBetweenAttempts' + // between each attempt, returning 'nil' the first time that 'fn' + // returns nil. If 'nil' is never returned, then the final error + // returned by 'fn' is returned. + Retry(tries int, durationBetweenAttempts time.Duration, fn func() error) + } + + // TestingCommon is an interface that describes the common methods of + // testing.T and testing.B ensuring this testutil can be used in both + // contexts. + TestingCommon interface { + Log(args ...any) + Logf(format string, args ...any) + Error(args ...any) + Errorf(format string, args ...any) + Fatal(args ...any) + Fatalf(format string, args ...any) + Skip(args ...any) + Skipf(format string, args ...any) + SkipNow() + Skipped() bool + Helper() + Cleanup(f func()) + TempDir() string + Setenv(key, value string) + } + + impl struct { + TestingCommon + } +) + +func New(tc TestingCommon) TT { + return &impl{TestingCommon: tc} +} + +func (t impl) AssertContains(err error, target string) { + t.Helper() + if err == nil || !strings.Contains(err.Error(), target) { + t.Fatalf("err: %v != target: %v", err, target) + } +} + +func (t impl) AssertIs(err, target error) { + t.Helper() + t.AssertContains(err, target.Error()) +} + +func (t impl) FailAll(vs ...interface{}) { + t.Helper() + for _, v := range vs { + if err, ok := v.(error); ok && err == nil { + t.Fatal("should've failed") + } + } +} + +func (t impl) OK(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} + +func (t impl) OKAll(vs ...interface{}) { + t.Helper() + for _, v := range vs { + if err, ok := v.(error); ok && err != nil { + t.Fatal(err) + } + } +} + +func (t impl) Retry(tries int, durationBetweenAttempts time.Duration, fn func() error) { + t.Helper() + for i := 1; i < tries; i++ { + err := fn() + if err == nil { + return + } + time.Sleep(durationBetweenAttempts) + } + t.OK(fn()) +} diff --git a/worker/host_test.go b/worker/host_test.go index 8dd0567ff..80096888c 100644 --- a/worker/host_test.go +++ b/worker/host_test.go @@ -14,6 +14,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" "go.sia.tech/renterd/hostdb" + "go.sia.tech/renterd/testutils" "lukechampine.com/frand" ) @@ -25,15 +26,15 @@ type ( } testHostManager struct { - t test + tt testutils.TT mu sync.Mutex hosts map[types.PublicKey]*testHost } ) -func newTestHostManager(t test) *testHostManager { - return &testHostManager{t: t, hosts: make(map[types.PublicKey]*testHost)} +func newTestHostManager(t testutils.TestingCommon) *testHostManager { + return &testHostManager{tt: testutils.New(t), hosts: make(map[types.PublicKey]*testHost)} } func (hm *testHostManager) Host(hk types.PublicKey, fcid types.FileContractID, siamuxAddr string) Host { @@ -41,7 +42,7 @@ func (hm *testHostManager) Host(hk types.PublicKey, fcid types.FileContractID, s defer hm.mu.Unlock() if _, ok := hm.hosts[hk]; !ok { - hm.t.Fatal("host not found") + hm.tt.Fatal("host not found") } return hm.hosts[hk] } diff --git a/worker/worker_test.go b/worker/worker_test.go index 7b781bf0e..8e973a378 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -7,18 +7,15 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/testutils" "go.uber.org/zap" "golang.org/x/crypto/blake2b" "lukechampine.com/frand" ) type ( - test interface { - Fatal(...any) - } - testWorker struct { - t test + tt testutils.TT *worker cs *contractStoreMock @@ -32,7 +29,7 @@ type ( } ) -func newTestWorker(t test) *testWorker { +func newTestWorker(t testutils.TestingCommon) *testWorker { // create bus dependencies cs := newContractStoreMock() os := newObjectStoreMock(testBucket) @@ -58,7 +55,7 @@ func newTestWorker(t test) *testWorker { w.uploadManager.mm = ulmm return &testWorker{ - t, + testutils.New(t), w, cs, os, @@ -88,7 +85,7 @@ func (w *testWorker) blockUploads() func() { select { case <-w.ulmm.memBlockChan: case <-time.After(time.Second): - w.t.Fatal("already blocking") + w.tt.Fatal("already blocking") } blockChan := make(chan struct{}) @@ -99,7 +96,7 @@ func (w *testWorker) blockUploads() func() { func (w *testWorker) contracts() []api.ContractMetadata { metadatas, err := w.cs.Contracts(context.Background(), api.ContractsOpts{}) if err != nil { - w.t.Fatal(err) + w.tt.Fatal(err) } return metadatas } @@ -107,12 +104,12 @@ func (w *testWorker) contracts() []api.ContractMetadata { func (w *testWorker) renewContract(hk types.PublicKey) *contractMock { h := w.hm.hosts[hk] if h == nil { - w.t.Fatal("host not found") + w.tt.Fatal("host not found") } renewal, err := w.cs.renewContract(hk) if err != nil { - w.t.Fatal(err) + w.tt.Fatal(err) } return renewal } From 3a15760bbcb8ebaf60a60ffad4da35f33a33eb4f Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 27 Feb 2024 15:29:19 +0100 Subject: [PATCH 07/12] worker: expose methods --- worker/bench_test.go | 14 +++++------ worker/downloader_test.go | 4 +-- worker/upload_test.go | 52 +++++++++++++++++++-------------------- worker/uploader_test.go | 4 +-- worker/worker_test.go | 12 ++++----- 5 files changed, 43 insertions(+), 43 deletions(-) diff --git a/worker/bench_test.go b/worker/bench_test.go index d6750c8f8..cc0034415 100644 --- a/worker/bench_test.go +++ b/worker/bench_test.go @@ -30,10 +30,10 @@ func BenchmarkDownloaderSingleObject(b *testing.B) { up.rs.MinShards = 10 up.rs.TotalShards = 30 up.packing = false - w.addHosts(up.rs.TotalShards) + w.AddHosts(up.rs.TotalShards) data := bytes.NewReader(frand.Bytes(int(up.rs.SlabSizeNoRedundancy()))) - _, _, err := w.uploadManager.Upload(context.Background(), data, w.contracts(), up, lockingPriorityUpload) + _, _, err := w.uploadManager.Upload(context.Background(), data, w.Contracts(), up, lockingPriorityUpload) if err != nil { b.Fatal(err) } @@ -45,7 +45,7 @@ func BenchmarkDownloaderSingleObject(b *testing.B) { b.SetBytes(o.Object.Size) b.ResetTimer() for i := 0; i < b.N; i++ { - err = w.downloadManager.DownloadObject(context.Background(), io.Discard, *o.Object.Object, 0, uint64(o.Object.Size), w.contracts()) + err = w.downloadManager.DownloadObject(context.Background(), io.Discard, *o.Object.Object, 0, uint64(o.Object.Size), w.Contracts()) if err != nil { b.Fatal(err) } @@ -63,13 +63,13 @@ func BenchmarkUploaderSingleObject(b *testing.B) { up.rs.MinShards = 10 up.rs.TotalShards = 30 up.packing = false - w.addHosts(up.rs.TotalShards) + w.AddHosts(up.rs.TotalShards) data := io.LimitReader(&zeroReader{}, int64(b.N*rhpv2.SectorSize*up.rs.MinShards)) b.SetBytes(int64(rhpv2.SectorSize * up.rs.MinShards)) b.ResetTimer() - _, _, err := w.uploadManager.Upload(context.Background(), data, w.contracts(), up, lockingPriorityUpload) + _, _, err := w.uploadManager.Upload(context.Background(), data, w.Contracts(), up, lockingPriorityUpload) if err != nil { b.Fatal(err) } @@ -86,14 +86,14 @@ func BenchmarkUploaderMultiObject(b *testing.B) { up.rs.MinShards = 10 up.rs.TotalShards = 30 up.packing = false - w.addHosts(up.rs.TotalShards) + w.AddHosts(up.rs.TotalShards) b.SetBytes(int64(rhpv2.SectorSize * up.rs.MinShards)) b.ResetTimer() for i := 0; i < b.N; i++ { data := io.LimitReader(&zeroReader{}, int64(rhpv2.SectorSize*up.rs.MinShards)) - _, _, err := w.uploadManager.Upload(context.Background(), data, w.contracts(), up, lockingPriorityUpload) + _, _, err := w.uploadManager.Upload(context.Background(), data, w.Contracts(), up, lockingPriorityUpload) if err != nil { b.Fatal(err) } diff --git a/worker/downloader_test.go b/worker/downloader_test.go index cbb48132c..8097b8304 100644 --- a/worker/downloader_test.go +++ b/worker/downloader_test.go @@ -8,13 +8,13 @@ import ( func TestDownloaderStopped(t *testing.T) { w := newTestWorker(t) - hosts := w.addHosts(1) + hosts := w.AddHosts(1) // convenience variables dm := w.downloadManager h := hosts[0] - dm.refreshDownloaders(w.contracts()) + dm.refreshDownloaders(w.Contracts()) dl := w.downloadManager.downloaders[h.PublicKey()] dl.Stop() diff --git a/worker/upload_test.go b/worker/upload_test.go index cc0996519..9192e040f 100644 --- a/worker/upload_test.go +++ b/worker/upload_test.go @@ -25,7 +25,7 @@ func TestUpload(t *testing.T) { w := newTestWorker(t) // add hosts to worker - w.addHosts(testRedundancySettings.TotalShards * 2) + w.AddHosts(testRedundancySettings.TotalShards * 2) // convenience variables os := w.os @@ -42,7 +42,7 @@ func TestUpload(t *testing.T) { params := testParameters(t.Name()) // upload data - _, _, err := ul.Upload(context.Background(), bytes.NewReader(data), w.contracts(), params, lockingPriorityUpload) + _, _, err := ul.Upload(context.Background(), bytes.NewReader(data), w.Contracts(), params, lockingPriorityUpload) if err != nil { t.Fatal(err) } @@ -61,7 +61,7 @@ func TestUpload(t *testing.T) { // download the data and assert it matches var buf bytes.Buffer - err = dl.DownloadObject(context.Background(), &buf, *o.Object.Object, 0, uint64(o.Object.Size), w.contracts()) + err = dl.DownloadObject(context.Background(), &buf, *o.Object.Object, 0, uint64(o.Object.Size), w.Contracts()) if err != nil { t.Fatal(err) } else if !bytes.Equal(data, buf.Bytes()) { @@ -71,7 +71,7 @@ func TestUpload(t *testing.T) { // filter contracts to have (at most) min shards used contracts var n int var filtered []api.ContractMetadata - for _, md := range w.contracts() { + for _, md := range w.Contracts() { // add unused contracts if _, used := used[md.HostKey]; !used { filtered = append(filtered, md) @@ -111,7 +111,7 @@ func TestUpload(t *testing.T) { // try and upload into a bucket that does not exist params.bucket = "doesnotexist" - _, _, err = ul.Upload(context.Background(), bytes.NewReader(data), w.contracts(), params, lockingPriorityUpload) + _, _, err = ul.Upload(context.Background(), bytes.NewReader(data), w.Contracts(), params, lockingPriorityUpload) if !errors.Is(err, api.ErrBucketNotFound) { t.Fatal("expected bucket not found error", err) } @@ -119,7 +119,7 @@ func TestUpload(t *testing.T) { // upload data using a cancelled context - assert we don't hang ctx, cancel := context.WithCancel(context.Background()) cancel() - _, _, err = ul.Upload(ctx, bytes.NewReader(data), w.contracts(), params, lockingPriorityUpload) + _, _, err = ul.Upload(ctx, bytes.NewReader(data), w.Contracts(), params, lockingPriorityUpload) if err == nil || !errors.Is(err, errUploadInterrupted) { t.Fatal(err) } @@ -130,7 +130,7 @@ func TestUploadPackedSlab(t *testing.T) { w := newTestWorker(t) // add hosts to worker - w.addHosts(testRedundancySettings.TotalShards * 2) + w.AddHosts(testRedundancySettings.TotalShards * 2) // convenience variables os := w.os @@ -149,7 +149,7 @@ func TestUploadPackedSlab(t *testing.T) { params.packing = true // upload data - _, _, err := ul.Upload(context.Background(), bytes.NewReader(data), w.contracts(), params, lockingPriorityUpload) + _, _, err := ul.Upload(context.Background(), bytes.NewReader(data), w.Contracts(), params, lockingPriorityUpload) if err != nil { t.Fatal(err) } @@ -167,7 +167,7 @@ func TestUploadPackedSlab(t *testing.T) { // download the data and assert it matches var buf bytes.Buffer - err = dl.DownloadObject(context.Background(), &buf, *o.Object.Object, 0, uint64(o.Object.Size), w.contracts()) + err = dl.DownloadObject(context.Background(), &buf, *o.Object.Object, 0, uint64(o.Object.Size), w.Contracts()) if err != nil { t.Fatal(err) } else if !bytes.Equal(data, buf.Bytes()) { @@ -185,7 +185,7 @@ func TestUploadPackedSlab(t *testing.T) { mem := mm.AcquireMemory(context.Background(), uint64(params.rs.TotalShards*rhpv2.SectorSize)) // upload the packed slab - err = ul.UploadPackedSlab(context.Background(), params.rs, ps, mem, w.contracts(), 0, lockingPriorityUpload) + err = ul.UploadPackedSlab(context.Background(), params.rs, ps, mem, w.Contracts(), 0, lockingPriorityUpload) if err != nil { t.Fatal(err) } @@ -203,7 +203,7 @@ func TestUploadPackedSlab(t *testing.T) { // download the data again and assert it matches buf.Reset() - err = dl.DownloadObject(context.Background(), &buf, *o.Object.Object, 0, uint64(o.Object.Size), w.contracts()) + err = dl.DownloadObject(context.Background(), &buf, *o.Object.Object, 0, uint64(o.Object.Size), w.Contracts()) if err != nil { t.Fatal(err) } else if !bytes.Equal(data, buf.Bytes()) { @@ -216,7 +216,7 @@ func TestUploadShards(t *testing.T) { w := newTestWorker(t) // add hosts to worker - w.addHosts(testRedundancySettings.TotalShards * 2) + w.AddHosts(testRedundancySettings.TotalShards * 2) // convenience variables os := w.os @@ -234,7 +234,7 @@ func TestUploadShards(t *testing.T) { params := testParameters(t.Name()) // upload data - _, _, err := ul.Upload(context.Background(), bytes.NewReader(data), w.contracts(), params, lockingPriorityUpload) + _, _, err := ul.Upload(context.Background(), bytes.NewReader(data), w.Contracts(), params, lockingPriorityUpload) if err != nil { t.Fatal(err) } @@ -265,7 +265,7 @@ func TestUploadShards(t *testing.T) { } // download the slab - shards, _, err := dl.DownloadSlab(context.Background(), slab.Slab, w.contracts()) + shards, _, err := dl.DownloadSlab(context.Background(), slab.Slab, w.Contracts()) if err != nil { t.Fatal(err) } @@ -281,7 +281,7 @@ func TestUploadShards(t *testing.T) { // recreate upload contracts contracts := make([]api.ContractMetadata, 0) - for _, c := range w.contracts() { + for _, c := range w.Contracts() { _, used := usedHosts[c.HostKey] _, bad := badHosts[c.HostKey] if !used && !bad { @@ -314,7 +314,7 @@ func TestUploadShards(t *testing.T) { // create download contracts contracts = contracts[:0] - for _, c := range w.contracts() { + for _, c := range w.Contracts() { if _, bad := badHosts[c.HostKey]; !bad { contracts = append(contracts, c) } @@ -335,7 +335,7 @@ func TestRefreshUploaders(t *testing.T) { w := newTestWorker(t) // add hosts to worker - w.addHosts(testRedundancySettings.TotalShards) + w.AddHosts(testRedundancySettings.TotalShards) // convenience variables ul := w.uploadManager @@ -352,7 +352,7 @@ func TestRefreshUploaders(t *testing.T) { params := testParameters(t.Name()) // upload data - contracts := w.contracts() + contracts := w.Contracts() _, err := w.upload(context.Background(), bytes.NewReader(data), contracts, params) if err != nil { t.Fatal(err) @@ -365,7 +365,7 @@ func TestRefreshUploaders(t *testing.T) { // renew the first contract c1 := contracts[0] - c1Renewed := w.renewContract(c1.HostKey) + c1Renewed := w.RenewContract(c1.HostKey) // remove the host from the second contract c2 := contracts[1] @@ -373,10 +373,10 @@ func TestRefreshUploaders(t *testing.T) { delete(cs.contracts, c2.ID) // add a new host/contract - hNew := w.addHost() + hNew := w.AddHost() // upload data - contracts = w.contracts() + contracts = w.Contracts() _, _, err = ul.Upload(context.Background(), bytes.NewReader(data), contracts, params, lockingPriorityUpload) if err != nil { t.Fatal(err) @@ -437,7 +437,7 @@ func TestUploadRegression(t *testing.T) { w := newTestWorker(t) // add hosts to worker - w.addHosts(testRedundancySettings.TotalShards) + w.AddHosts(testRedundancySettings.TotalShards) // convenience variables os := w.os @@ -453,12 +453,12 @@ func TestUploadRegression(t *testing.T) { params := testParameters(t.Name()) // make sure the memory manager blocks - unblock := w.blockUploads() + unblock := w.BlockUploads() // upload data ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - _, err := w.upload(ctx, bytes.NewReader(data), w.contracts(), params) + _, err := w.upload(ctx, bytes.NewReader(data), w.Contracts(), params) if !errors.Is(err, errUploadInterrupted) { t.Fatal(err) } @@ -467,7 +467,7 @@ func TestUploadRegression(t *testing.T) { unblock() // upload data - _, err = w.upload(context.Background(), bytes.NewReader(data), w.contracts(), params) + _, err = w.upload(context.Background(), bytes.NewReader(data), w.Contracts(), params) if err != nil { t.Fatal(err) } @@ -480,7 +480,7 @@ func TestUploadRegression(t *testing.T) { // download data for good measure var buf bytes.Buffer - err = dl.DownloadObject(context.Background(), &buf, *o.Object.Object, 0, uint64(o.Object.Size), w.contracts()) + err = dl.DownloadObject(context.Background(), &buf, *o.Object.Object, 0, uint64(o.Object.Size), w.Contracts()) if err != nil { t.Fatal(err) } else if !bytes.Equal(data, buf.Bytes()) { diff --git a/worker/uploader_test.go b/worker/uploader_test.go index 514d17aab..b203827a5 100644 --- a/worker/uploader_test.go +++ b/worker/uploader_test.go @@ -9,10 +9,10 @@ import ( func TestUploaderStopped(t *testing.T) { w := newTestWorker(t) - w.addHosts(1) + w.AddHosts(1) um := w.uploadManager - um.refreshUploaders(w.contracts(), 1) + um.refreshUploaders(w.Contracts(), 1) ul := um.uploaders[0] ul.Stop(errors.New("test")) diff --git a/worker/worker_test.go b/worker/worker_test.go index 8e973a378..eda968b47 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -66,14 +66,14 @@ func newTestWorker(t testutils.TestingCommon) *testWorker { } } -func (w *testWorker) addHosts(n int) (added []*testHost) { +func (w *testWorker) AddHosts(n int) (added []*testHost) { for i := 0; i < n; i++ { - added = append(added, w.addHost()) + added = append(added, w.AddHost()) } return } -func (w *testWorker) addHost() *testHost { +func (w *testWorker) AddHost() *testHost { h := w.hs.addHost() c := w.cs.addContract(h.hk) host := newTestHost(h, c) @@ -81,7 +81,7 @@ func (w *testWorker) addHost() *testHost { return host } -func (w *testWorker) blockUploads() func() { +func (w *testWorker) BlockUploads() func() { select { case <-w.ulmm.memBlockChan: case <-time.After(time.Second): @@ -93,7 +93,7 @@ func (w *testWorker) blockUploads() func() { return func() { close(blockChan) } } -func (w *testWorker) contracts() []api.ContractMetadata { +func (w *testWorker) Contracts() []api.ContractMetadata { metadatas, err := w.cs.Contracts(context.Background(), api.ContractsOpts{}) if err != nil { w.tt.Fatal(err) @@ -101,7 +101,7 @@ func (w *testWorker) contracts() []api.ContractMetadata { return metadatas } -func (w *testWorker) renewContract(hk types.PublicKey) *contractMock { +func (w *testWorker) RenewContract(hk types.PublicKey) *contractMock { h := w.hm.hosts[hk] if h == nil { w.tt.Fatal("host not found") From 31198abe40359ed2edbc1c83cd896594b328618d Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 27 Feb 2024 16:38:07 +0100 Subject: [PATCH 08/12] testing: move tt.go --- internal/{testing => test/e2e}/blocklist_test.go | 2 +- internal/{testing => test/e2e}/cluster.go | 8 ++++---- internal/{testing => test/e2e}/cluster_test.go | 2 +- internal/{testing => test/e2e}/gouging_test.go | 2 +- internal/{testing => test/e2e}/host.go | 2 +- internal/{testing => test/e2e}/interactions_test.go | 2 +- internal/{testing => test/e2e}/metadata_test.go | 2 +- internal/{testing => test/e2e}/metrics_test.go | 2 +- internal/{testing => test/e2e}/migrations_test.go | 2 +- internal/{testing => test/e2e}/pruning_test.go | 2 +- internal/{testing => test/e2e}/s3_test.go | 2 +- internal/{testing => test/e2e}/uploads_test.go | 2 +- {testutils => internal/test}/tt.go | 4 ++-- worker/host_test.go | 8 ++++---- worker/worker_test.go | 8 ++++---- 15 files changed, 25 insertions(+), 25 deletions(-) rename internal/{testing => test/e2e}/blocklist_test.go (99%) rename internal/{testing => test/e2e}/cluster.go (99%) rename internal/{testing => test/e2e}/cluster_test.go (99%) rename internal/{testing => test/e2e}/gouging_test.go (99%) rename internal/{testing => test/e2e}/host.go (99%) rename internal/{testing => test/e2e}/interactions_test.go (99%) rename internal/{testing => test/e2e}/metadata_test.go (99%) rename internal/{testing => test/e2e}/metrics_test.go (99%) rename internal/{testing => test/e2e}/migrations_test.go (99%) rename internal/{testing => test/e2e}/pruning_test.go (99%) rename internal/{testing => test/e2e}/s3_test.go (99%) rename internal/{testing => test/e2e}/uploads_test.go (99%) rename {testutils => internal/test}/tt.go (97%) diff --git a/internal/testing/blocklist_test.go b/internal/test/e2e/blocklist_test.go similarity index 99% rename from internal/testing/blocklist_test.go rename to internal/test/e2e/blocklist_test.go index 9d9a12605..48358b854 100644 --- a/internal/testing/blocklist_test.go +++ b/internal/test/e2e/blocklist_test.go @@ -1,4 +1,4 @@ -package testing +package e2e import ( "context" diff --git a/internal/testing/cluster.go b/internal/test/e2e/cluster.go similarity index 99% rename from internal/testing/cluster.go rename to internal/test/e2e/cluster.go index d36f143be..962aa11ef 100644 --- a/internal/testing/cluster.go +++ b/internal/test/e2e/cluster.go @@ -1,4 +1,4 @@ -package testing +package e2e import ( "context" @@ -24,9 +24,9 @@ import ( "go.sia.tech/renterd/bus" "go.sia.tech/renterd/config" "go.sia.tech/renterd/internal/node" + "go.sia.tech/renterd/internal/test" "go.sia.tech/renterd/s3" "go.sia.tech/renterd/stores" - "go.sia.tech/renterd/testutils" "go.uber.org/zap" "go.uber.org/zap/zapcore" "gorm.io/gorm" @@ -120,7 +120,7 @@ type TestCluster struct { dbName string dir string logger *zap.Logger - tt testutils.TT + tt test.TT wk types.PrivateKey wg sync.WaitGroup } @@ -245,7 +245,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { if testing.Short() { t.SkipNow() } - tt := testutils.New(t) + tt := test.NewTest(t) // Ensure we don't hang ctx, cancel := context.WithTimeout(context.Background(), time.Minute) diff --git a/internal/testing/cluster_test.go b/internal/test/e2e/cluster_test.go similarity index 99% rename from internal/testing/cluster_test.go rename to internal/test/e2e/cluster_test.go index d7c458d8b..18d12fa48 100644 --- a/internal/testing/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -1,4 +1,4 @@ -package testing +package e2e import ( "bytes" diff --git a/internal/testing/gouging_test.go b/internal/test/e2e/gouging_test.go similarity index 99% rename from internal/testing/gouging_test.go rename to internal/test/e2e/gouging_test.go index 7a812354f..423733ab6 100644 --- a/internal/testing/gouging_test.go +++ b/internal/test/e2e/gouging_test.go @@ -1,4 +1,4 @@ -package testing +package e2e import ( "bytes" diff --git a/internal/testing/host.go b/internal/test/e2e/host.go similarity index 99% rename from internal/testing/host.go rename to internal/test/e2e/host.go index e7943a7d3..6100adad5 100644 --- a/internal/testing/host.go +++ b/internal/test/e2e/host.go @@ -1,4 +1,4 @@ -package testing +package e2e import ( "context" diff --git a/internal/testing/interactions_test.go b/internal/test/e2e/interactions_test.go similarity index 99% rename from internal/testing/interactions_test.go rename to internal/test/e2e/interactions_test.go index 686003e02..021d75cb6 100644 --- a/internal/testing/interactions_test.go +++ b/internal/test/e2e/interactions_test.go @@ -1,4 +1,4 @@ -package testing +package e2e import ( "context" diff --git a/internal/testing/metadata_test.go b/internal/test/e2e/metadata_test.go similarity index 99% rename from internal/testing/metadata_test.go rename to internal/test/e2e/metadata_test.go index c88c8650d..47fb6a2b1 100644 --- a/internal/testing/metadata_test.go +++ b/internal/test/e2e/metadata_test.go @@ -1,4 +1,4 @@ -package testing +package e2e import ( "bytes" diff --git a/internal/testing/metrics_test.go b/internal/test/e2e/metrics_test.go similarity index 99% rename from internal/testing/metrics_test.go rename to internal/test/e2e/metrics_test.go index 7dd0195f5..ed432c3c2 100644 --- a/internal/testing/metrics_test.go +++ b/internal/test/e2e/metrics_test.go @@ -1,4 +1,4 @@ -package testing +package e2e import ( "bytes" diff --git a/internal/testing/migrations_test.go b/internal/test/e2e/migrations_test.go similarity index 99% rename from internal/testing/migrations_test.go rename to internal/test/e2e/migrations_test.go index 2afbcebb6..66325cc15 100644 --- a/internal/testing/migrations_test.go +++ b/internal/test/e2e/migrations_test.go @@ -1,4 +1,4 @@ -package testing +package e2e import ( "bytes" diff --git a/internal/testing/pruning_test.go b/internal/test/e2e/pruning_test.go similarity index 99% rename from internal/testing/pruning_test.go rename to internal/test/e2e/pruning_test.go index 80e6ab29d..bd1bb822d 100644 --- a/internal/testing/pruning_test.go +++ b/internal/test/e2e/pruning_test.go @@ -1,4 +1,4 @@ -package testing +package e2e import ( "bytes" diff --git a/internal/testing/s3_test.go b/internal/test/e2e/s3_test.go similarity index 99% rename from internal/testing/s3_test.go rename to internal/test/e2e/s3_test.go index ced1fbcc0..92b37af04 100644 --- a/internal/testing/s3_test.go +++ b/internal/test/e2e/s3_test.go @@ -1,4 +1,4 @@ -package testing +package e2e import ( "bytes" diff --git a/internal/testing/uploads_test.go b/internal/test/e2e/uploads_test.go similarity index 99% rename from internal/testing/uploads_test.go rename to internal/test/e2e/uploads_test.go index e3e938120..96daae8c8 100644 --- a/internal/testing/uploads_test.go +++ b/internal/test/e2e/uploads_test.go @@ -1,4 +1,4 @@ -package testing +package e2e import ( "bytes" diff --git a/testutils/tt.go b/internal/test/tt.go similarity index 97% rename from testutils/tt.go rename to internal/test/tt.go index 9345a2fd7..c48e2616b 100644 --- a/testutils/tt.go +++ b/internal/test/tt.go @@ -1,4 +1,4 @@ -package testutils +package test import ( "strings" @@ -47,7 +47,7 @@ type ( } ) -func New(tc TestingCommon) TT { +func NewTest(tc TestingCommon) TT { return &impl{TestingCommon: tc} } diff --git a/worker/host_test.go b/worker/host_test.go index 80096888c..af4052507 100644 --- a/worker/host_test.go +++ b/worker/host_test.go @@ -14,7 +14,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" "go.sia.tech/renterd/hostdb" - "go.sia.tech/renterd/testutils" + "go.sia.tech/renterd/internal/test" "lukechampine.com/frand" ) @@ -26,15 +26,15 @@ type ( } testHostManager struct { - tt testutils.TT + tt test.TT mu sync.Mutex hosts map[types.PublicKey]*testHost } ) -func newTestHostManager(t testutils.TestingCommon) *testHostManager { - return &testHostManager{tt: testutils.New(t), hosts: make(map[types.PublicKey]*testHost)} +func newTestHostManager(t test.TestingCommon) *testHostManager { + return &testHostManager{tt: test.NewTest(t), hosts: make(map[types.PublicKey]*testHost)} } func (hm *testHostManager) Host(hk types.PublicKey, fcid types.FileContractID, siamuxAddr string) Host { diff --git a/worker/worker_test.go b/worker/worker_test.go index eda968b47..bef0214ac 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -7,7 +7,7 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/testutils" + "go.sia.tech/renterd/internal/test" "go.uber.org/zap" "golang.org/x/crypto/blake2b" "lukechampine.com/frand" @@ -15,7 +15,7 @@ import ( type ( testWorker struct { - tt testutils.TT + tt test.TT *worker cs *contractStoreMock @@ -29,7 +29,7 @@ type ( } ) -func newTestWorker(t testutils.TestingCommon) *testWorker { +func newTestWorker(t test.TestingCommon) *testWorker { // create bus dependencies cs := newContractStoreMock() os := newObjectStoreMock(testBucket) @@ -55,7 +55,7 @@ func newTestWorker(t testutils.TestingCommon) *testWorker { w.uploadManager.mm = ulmm return &testWorker{ - testutils.New(t), + test.NewTest(t), w, cs, os, From 30ced3e5a8decd1c074b9ea834ce63340b32a128 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 27 Feb 2024 16:41:47 +0100 Subject: [PATCH 09/12] ci: update test.yml --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e8a32e5ec..b3c794242 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,7 @@ jobs: - name: Test Integration uses: n8maninger/action-golang-test@v1 with: - package: "./internal/testing/..." + package: "./internal/test/..." args: "-failfast;-race;-tags=testing;-timeout=30m" - name: Test Integration - MySQL if: matrix.os == 'ubuntu-latest' @@ -70,7 +70,7 @@ jobs: RENTERD_DB_USER: root RENTERD_DB_PASSWORD: test with: - package: "./internal/testing/..." + package: "./internal/test/..." args: "-failfast;-race;-tags=testing;-timeout=30m" - name: Build run: go build -o bin/ ./cmd/renterd From dfc7850c3e8cf5a5a2088d7db6451a8beb9336a8 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 27 Feb 2024 16:47:27 +0100 Subject: [PATCH 10/12] ci: fix e2e path --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b3c794242..bb56bbdb2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,7 @@ jobs: - name: Test Integration uses: n8maninger/action-golang-test@v1 with: - package: "./internal/test/..." + package: "./internal/test/e2e/..." args: "-failfast;-race;-tags=testing;-timeout=30m" - name: Test Integration - MySQL if: matrix.os == 'ubuntu-latest' @@ -70,7 +70,7 @@ jobs: RENTERD_DB_USER: root RENTERD_DB_PASSWORD: test with: - package: "./internal/test/..." + package: "./internal/test/e2e/..." args: "-failfast;-race;-tags=testing;-timeout=30m" - name: Build run: go build -o bin/ ./cmd/renterd From 990feac6ea56041c265790f65c7d3fe9dd84bf75 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 27 Feb 2024 16:48:34 +0100 Subject: [PATCH 11/12] internal: rename TT constr --- internal/test/e2e/cluster.go | 2 +- internal/test/tt.go | 2 +- worker/host_test.go | 2 +- worker/worker_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 962aa11ef..142bedf47 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -245,7 +245,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { if testing.Short() { t.SkipNow() } - tt := test.NewTest(t) + tt := test.NewTT(t) // Ensure we don't hang ctx, cancel := context.WithTimeout(context.Background(), time.Minute) diff --git a/internal/test/tt.go b/internal/test/tt.go index c48e2616b..22bcff223 100644 --- a/internal/test/tt.go +++ b/internal/test/tt.go @@ -47,7 +47,7 @@ type ( } ) -func NewTest(tc TestingCommon) TT { +func NewTT(tc TestingCommon) TT { return &impl{TestingCommon: tc} } diff --git a/worker/host_test.go b/worker/host_test.go index af4052507..618c4cfb3 100644 --- a/worker/host_test.go +++ b/worker/host_test.go @@ -34,7 +34,7 @@ type ( ) func newTestHostManager(t test.TestingCommon) *testHostManager { - return &testHostManager{tt: test.NewTest(t), hosts: make(map[types.PublicKey]*testHost)} + return &testHostManager{tt: test.NewTT(t), hosts: make(map[types.PublicKey]*testHost)} } func (hm *testHostManager) Host(hk types.PublicKey, fcid types.FileContractID, siamuxAddr string) Host { diff --git a/worker/worker_test.go b/worker/worker_test.go index bef0214ac..5e6a1554f 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -55,7 +55,7 @@ func newTestWorker(t test.TestingCommon) *testWorker { w.uploadManager.mm = ulmm return &testWorker{ - test.NewTest(t), + test.NewTT(t), w, cs, os, From 9798d6410613c61475926480b43e3c99ff2f879a Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 27 Feb 2024 16:55:44 +0100 Subject: [PATCH 12/12] testing: add config.go --- internal/test/config.go | 64 ++++++++++++++++++++++ internal/test/e2e/blocklist_test.go | 9 ++-- internal/test/e2e/cluster.go | 80 +++++----------------------- internal/test/e2e/cluster_test.go | 67 +++++++++++------------ internal/test/e2e/gouging_test.go | 5 +- internal/test/e2e/metadata_test.go | 3 +- internal/test/e2e/metrics_test.go | 5 +- internal/test/e2e/migrations_test.go | 11 ++-- internal/test/e2e/pruning_test.go | 5 +- internal/test/e2e/s3_test.go | 17 +++--- internal/test/e2e/uploads_test.go | 5 +- 11 files changed, 145 insertions(+), 126 deletions(-) create mode 100644 internal/test/config.go diff --git a/internal/test/config.go b/internal/test/config.go new file mode 100644 index 000000000..7553fa16d --- /dev/null +++ b/internal/test/config.go @@ -0,0 +1,64 @@ +package test + +import ( + "time" + + "github.com/minio/minio-go/v7/pkg/credentials" + rhpv2 "go.sia.tech/core/rhp/v2" + "go.sia.tech/core/types" + "go.sia.tech/renterd/api" +) + +var ( + // AutopilotConfig is the autopilot used for testing unless a different + // one is explicitly set. + AutopilotConfig = api.AutopilotConfig{ + Contracts: api.ContractsConfig{ + Allowance: types.Siacoins(1).Mul64(1e3), + Amount: 3, + Period: 144, + RenewWindow: 72, + + Download: rhpv2.SectorSize * 500, + Upload: rhpv2.SectorSize * 500, + Storage: rhpv2.SectorSize * 5e3, + + Set: ContractSet, + Prune: false, + }, + Hosts: api.HostsConfig{ + MaxDowntimeHours: 10, + MinRecentScanFailures: 10, + AllowRedundantIPs: true, // allow for integration tests by default + }, + } + + ContractSet = "testset" + ContractSetSettings = api.ContractSetSetting{ + Default: ContractSet, + } + + GougingSettings = api.GougingSettings{ + MinMaxCollateral: types.Siacoins(10), // at least up to 10 SC per contract + MaxRPCPrice: types.Siacoins(1).Div64(1000), // 1mS per RPC + MaxContractPrice: types.Siacoins(10), // 10 SC per contract + MaxDownloadPrice: types.Siacoins(1).Mul64(1000), // 1000 SC per 1 TiB + MaxUploadPrice: types.Siacoins(1).Mul64(1000), // 1000 SC per 1 TiB + MaxStoragePrice: types.Siacoins(1000).Div64(144 * 30), // 1000 SC per month + + HostBlockHeightLeeway: 240, // amount of leeway given to host block height + + MinPriceTableValidity: 10 * time.Second, // minimum value for price table validity + MinAccountExpiry: time.Hour, // minimum value for account expiry + MinMaxEphemeralAccountBalance: types.Siacoins(1), // 1SC + } + + RedundancySettings = api.RedundancySettings{ + MinShards: 2, + TotalShards: 3, + } + + S3AccessKeyID = "TESTINGYNHUWCPKOPSYQ" + S3SecretAccessKey = "Rh30BNyj+qNI4ftYRteoZbHJ3X4Ln71QtZkRXzJ9" + S3Credentials = credentials.NewStaticV4(S3AccessKeyID, S3SecretAccessKey, "") +) diff --git a/internal/test/e2e/blocklist_test.go b/internal/test/e2e/blocklist_test.go index 48358b854..64acc2fba 100644 --- a/internal/test/e2e/blocklist_test.go +++ b/internal/test/e2e/blocklist_test.go @@ -8,6 +8,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/test" ) func TestBlocklist(t *testing.T) { @@ -26,7 +27,7 @@ func TestBlocklist(t *testing.T) { tt := cluster.tt // fetch contracts - contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: testAutopilotConfig.Contracts.Set}) + contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: test.AutopilotConfig.Contracts.Set}) tt.OK(err) if len(contracts) != 3 { t.Fatalf("unexpected number of contracts, %v != 3", len(contracts)) @@ -40,7 +41,7 @@ func TestBlocklist(t *testing.T) { // assert h3 is no longer in the contract set tt.Retry(5, time.Second, func() error { - contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: testAutopilotConfig.Contracts.Set}) + contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: test.AutopilotConfig.Contracts.Set}) tt.OK(err) if len(contracts) != 2 { return fmt.Errorf("unexpected number of contracts, %v != 2", len(contracts)) @@ -60,7 +61,7 @@ func TestBlocklist(t *testing.T) { // assert h1 is no longer in the contract set tt.Retry(5, time.Second, func() error { - contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: testAutopilotConfig.Contracts.Set}) + contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: test.AutopilotConfig.Contracts.Set}) tt.OK(err) if len(contracts) != 1 { return fmt.Errorf("unexpected number of contracts, %v != 1", len(contracts)) @@ -77,7 +78,7 @@ func TestBlocklist(t *testing.T) { tt.OK(b.UpdateHostAllowlist(ctx, nil, []types.PublicKey{hk1, hk2}, false)) tt.OK(b.UpdateHostBlocklist(ctx, nil, []string{h1.NetAddress}, false)) tt.Retry(5, time.Second, func() error { - contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: testAutopilotConfig.Contracts.Set}) + contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: test.AutopilotConfig.Contracts.Set}) tt.OK(err) if len(contracts) != 3 { return fmt.Errorf("unexpected number of contracts, %v != 3", len(contracts)) diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 142bedf47..b9776d598 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -14,9 +14,7 @@ import ( "time" "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" "go.sia.tech/core/consensus" - rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" "go.sia.tech/jape" "go.sia.tech/renterd/api" @@ -36,66 +34,14 @@ import ( ) const ( - testBusFlushInterval = 100 * time.Millisecond - testContractSet = "testset" - testPersistInterval = 2 * time.Second - latestHardforkHeight = 50 // foundation hardfork height in testing + testBusFlushInterval = 100 * time.Millisecond + testBusPersistInterval = 2 * time.Second + latestHardforkHeight = 50 // foundation hardfork height in testing ) var ( clusterOptsDefault = testClusterOptions{} clusterOptNoFunding = false - - // testAutopilotConfig is the autopilot used for testing unless a different - // one is explicitly set. - testAutopilotConfig = api.AutopilotConfig{ - Contracts: api.ContractsConfig{ - Allowance: types.Siacoins(1).Mul64(1e3), - Amount: 3, - Period: 144, - RenewWindow: 72, - - Download: rhpv2.SectorSize * 500, - Upload: rhpv2.SectorSize * 500, - Storage: rhpv2.SectorSize * 5e3, - - Set: testContractSet, - Prune: false, - }, - Hosts: api.HostsConfig{ - MaxDowntimeHours: 10, - MinRecentScanFailures: 10, - AllowRedundantIPs: true, // allow for integration tests by default - }, - } - - testContractSetSettings = api.ContractSetSetting{ - Default: testContractSet, - } - - testGougingSettings = api.GougingSettings{ - MinMaxCollateral: types.Siacoins(10), // at least up to 10 SC per contract - MaxRPCPrice: types.Siacoins(1).Div64(1000), // 1mS per RPC - MaxContractPrice: types.Siacoins(10), // 10 SC per contract - MaxDownloadPrice: types.Siacoins(1).Mul64(1000), // 1000 SC per 1 TiB - MaxUploadPrice: types.Siacoins(1).Mul64(1000), // 1000 SC per 1 TiB - MaxStoragePrice: types.Siacoins(1000).Div64(144 * 30), // 1000 SC per month - - HostBlockHeightLeeway: 240, // amount of leeway given to host block height - - MinPriceTableValidity: 10 * time.Second, // minimum value for price table validity - MinAccountExpiry: time.Hour, // minimum value for account expiry - MinMaxEphemeralAccountBalance: types.Siacoins(1), // 1SC - } - - testRedundancySettings = api.RedundancySettings{ - MinShards: 2, - TotalShards: 3, - } - - testS3AccessKeyID = "TESTINGYNHUWCPKOPSYQ" - testS3SecretAccessKey = "Rh30BNyj+qNI4ftYRteoZbHJ3X4Ln71QtZkRXzJ9" - testS3Credentials = credentials.NewStaticV4(testS3AccessKeyID, testS3SecretAccessKey, "") ) // TestCluster is a helper type that allows for easily creating a number of @@ -287,7 +233,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { if opts.uploadPacking { enableUploadPacking = opts.uploadPacking } - apSettings := testAutopilotConfig + apSettings := test.AutopilotConfig if opts.autopilotSettings != nil { apSettings = *opts.autopilotSettings } @@ -340,14 +286,14 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { busClient := bus.NewClient(busAddr, busPassword) workerClient := worker.NewClient(workerAddr, workerPassword) s3Client, err := minio.New(s3Addr, &minio.Options{ - Creds: testS3Credentials, + Creds: test.S3Credentials, Secure: false, }) tt.OK(err) url := s3Client.EndpointURL() s3Core, err := minio.NewCore(url.Host+url.Path, &minio.Options{ - Creds: testS3Credentials, + Creds: test.S3Credentials, }) tt.OK(err) @@ -455,7 +401,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { // Set the test contract set to make sure we can add objects at the // beginning of a test right away. - tt.OK(busClient.SetContractSet(ctx, testContractSet, []types.FileContractID{})) + tt.OK(busClient.SetContractSet(ctx, test.ContractSet, []types.FileContractID{})) // Update the autopilot to use test settings if !opts.skipSettingAutopilot { @@ -466,11 +412,11 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { } // Update the bus settings. - tt.OK(busClient.UpdateSetting(ctx, api.SettingGouging, testGougingSettings)) - tt.OK(busClient.UpdateSetting(ctx, api.SettingRedundancy, testRedundancySettings)) - tt.OK(busClient.UpdateSetting(ctx, api.SettingContractSet, testContractSetSettings)) + tt.OK(busClient.UpdateSetting(ctx, api.SettingGouging, test.GougingSettings)) + tt.OK(busClient.UpdateSetting(ctx, api.SettingRedundancy, test.RedundancySettings)) + tt.OK(busClient.UpdateSetting(ctx, api.SettingContractSet, test.ContractSetSettings)) tt.OK(busClient.UpdateSetting(ctx, api.SettingS3Authentication, api.S3AuthenticationSettings{ - V4Keypairs: map[string]string{testS3AccessKeyID: testS3SecretAccessKey}, + V4Keypairs: map[string]string{test.S3AccessKeyID: test.S3SecretAccessKey}, })) tt.OK(busClient.UpdateSetting(ctx, api.SettingUploadPacking, api.UploadPackingSettings{Enabled: enableUploadPacking})) @@ -501,7 +447,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { if nHosts > 0 { cluster.AddHostsBlocking(nHosts) cluster.WaitForContracts() - cluster.WaitForContractSet(testContractSet, nHosts) + cluster.WaitForContractSet(test.ContractSet, nHosts) _ = cluster.WaitForAccounts() } @@ -926,7 +872,7 @@ func testBusCfg() node.BusConfig { AnnouncementMaxAgeHours: 24 * 7 * 52, // 1 year Bootstrap: false, GatewayAddr: "127.0.0.1:0", - PersistInterval: testPersistInterval, + PersistInterval: testBusPersistInterval, UsedUTXOExpiry: time.Minute, SlabBufferCompletionThreshold: 0, }, diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 5e9f440db..c546937eb 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -24,6 +24,7 @@ import ( "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" "go.sia.tech/renterd/hostdb" + "go.sia.tech/renterd/internal/test" "go.sia.tech/renterd/object" "go.sia.tech/renterd/wallet" "go.uber.org/zap" @@ -264,7 +265,7 @@ func TestObjectEntries(t *testing.T) { // create a test cluster cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, }) defer cluster.Shutdown() @@ -435,7 +436,7 @@ func TestObjectsRename(t *testing.T) { // create a test cluster cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, }) defer cluster.Shutdown() @@ -491,7 +492,7 @@ func TestUploadDownloadEmpty(t *testing.T) { // create a test cluster cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, }) defer cluster.Shutdown() @@ -519,13 +520,13 @@ func TestUploadDownloadBasic(t *testing.T) { } // sanity check the default settings - if testAutopilotConfig.Contracts.Amount < uint64(testRedundancySettings.MinShards) { + if test.AutopilotConfig.Contracts.Amount < uint64(test.RedundancySettings.MinShards) { t.Fatal("too few hosts to support the redundancy settings") } // create a test cluster cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, }) defer cluster.Shutdown() @@ -546,8 +547,8 @@ func TestUploadDownloadBasic(t *testing.T) { for _, slab := range resp.Object.Slabs { hosts := make(map[types.PublicKey]struct{}) roots := make(map[types.Hash256]struct{}) - if len(slab.Shards) != testRedundancySettings.TotalShards { - t.Fatal("wrong amount of shards", len(slab.Shards), testRedundancySettings.TotalShards) + if len(slab.Shards) != test.RedundancySettings.TotalShards { + t.Fatal("wrong amount of shards", len(slab.Shards), test.RedundancySettings.TotalShards) } for _, shard := range slab.Shards { if shard.LatestHost == (types.PublicKey{}) { @@ -631,13 +632,13 @@ func TestUploadDownloadExtended(t *testing.T) { } // sanity check the default settings - if testAutopilotConfig.Contracts.Amount < uint64(testRedundancySettings.MinShards) { + if test.AutopilotConfig.Contracts.Amount < uint64(test.RedundancySettings.MinShards) { t.Fatal("too few hosts to support the redundancy settings") } // create a test cluster cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, }) defer cluster.Shutdown() @@ -771,18 +772,18 @@ func TestUploadDownloadExtended(t *testing.T) { // and download spending metrics are tracked properly. func TestUploadDownloadSpending(t *testing.T) { // sanity check the default settings - if testAutopilotConfig.Contracts.Amount < uint64(testRedundancySettings.MinShards) { + if test.AutopilotConfig.Contracts.Amount < uint64(test.RedundancySettings.MinShards) { t.Fatal("too few hosts to support the redundancy settings") } // create a test cluster cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, }) defer cluster.Shutdown() w := cluster.Worker - rs := testRedundancySettings + rs := test.RedundancySettings tt := cluster.tt // check that the funding was recorded @@ -891,7 +892,7 @@ func TestUploadDownloadSpending(t *testing.T) { } // fetch contract set contracts - contracts, err := cluster.Bus.Contracts(context.Background(), api.ContractsOpts{ContractSet: testAutopilotConfig.Contracts.Set}) + contracts, err := cluster.Bus.Contracts(context.Background(), api.ContractsOpts{ContractSet: test.AutopilotConfig.Contracts.Set}) tt.OK(err) currentSet := make(map[types.FileContractID]struct{}) for _, c := range contracts { @@ -1090,7 +1091,7 @@ func TestParallelUpload(t *testing.T) { // create a test cluster cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, }) defer cluster.Shutdown() @@ -1168,7 +1169,7 @@ func TestParallelDownload(t *testing.T) { // create a test cluster cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, }) defer cluster.Shutdown() @@ -1285,7 +1286,7 @@ func TestUploadDownloadSameHost(t *testing.T) { // create a test cluster cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, }) defer cluster.Shutdown() tt := cluster.tt @@ -1316,7 +1317,7 @@ func TestUploadDownloadSameHost(t *testing.T) { // build a frankenstein object constructed with all sectors on the same host res.Object.Slabs[0].Shards = shards[res.Object.Slabs[0].Shards[0].LatestHost] - tt.OK(b.AddObject(context.Background(), api.DefaultBucketName, "frankenstein", testContractSet, *res.Object.Object, api.AddObjectOptions{})) + tt.OK(b.AddObject(context.Background(), api.DefaultBucketName, "frankenstein", test.ContractSet, *res.Object.Object, api.AddObjectOptions{})) // assert we can download this object tt.OK(w.DownloadObject(context.Background(), io.Discard, api.DefaultBucketName, "frankenstein", api.DownloadObjectOptions{})) @@ -1524,20 +1525,20 @@ func TestUploadPacking(t *testing.T) { } // sanity check the default settings - if testAutopilotConfig.Contracts.Amount < uint64(testRedundancySettings.MinShards) { + if test.AutopilotConfig.Contracts.Amount < uint64(test.RedundancySettings.MinShards) { t.Fatal("too few hosts to support the redundancy settings") } // create a test cluster cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, uploadPacking: true, }) defer cluster.Shutdown() b := cluster.Bus w := cluster.Worker - rs := testRedundancySettings + rs := test.RedundancySettings tt := cluster.tt // prepare 3 files which are all smaller than a slab but together make up @@ -1772,7 +1773,7 @@ func TestSlabBufferStats(t *testing.T) { } // sanity check the default settings - if testAutopilotConfig.Contracts.Amount < uint64(testRedundancySettings.MinShards) { + if test.AutopilotConfig.Contracts.Amount < uint64(test.RedundancySettings.MinShards) { t.Fatal("too few hosts to support the redundancy settings") } @@ -1782,14 +1783,14 @@ func TestSlabBufferStats(t *testing.T) { busCfg.SlabBufferCompletionThreshold = int64(threshold) cluster := newTestCluster(t, testClusterOptions{ busCfg: &busCfg, - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, uploadPacking: true, }) defer cluster.Shutdown() b := cluster.Bus w := cluster.Worker - rs := testRedundancySettings + rs := test.RedundancySettings tt := cluster.tt // prepare 3 files which are all smaller than a slab but together make up @@ -1838,8 +1839,8 @@ func TestSlabBufferStats(t *testing.T) { if len(buffers) != 1 { t.Fatal("expected 1 slab buffer, got", len(buffers)) } - if buffers[0].ContractSet != testContractSet { - t.Fatalf("expected slab buffer contract set of %v, got %v", testContractSet, buffers[0].ContractSet) + if buffers[0].ContractSet != test.ContractSet { + t.Fatalf("expected slab buffer contract set of %v, got %v", test.ContractSet, buffers[0].ContractSet) } if buffers[0].Size != int64(len(data1)) { t.Fatalf("expected slab buffer size of %v, got %v", len(data1), buffers[0].Size) @@ -2020,7 +2021,7 @@ func TestMultipartUploads(t *testing.T) { } cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, uploadPacking: true, }) defer cluster.Shutdown() @@ -2266,7 +2267,7 @@ func TestWalletFormUnconfirmed(t *testing.T) { } // Enable autopilot by setting it. - cluster.UpdateAutopilotConfig(context.Background(), testAutopilotConfig) + cluster.UpdateAutopilotConfig(context.Background(), test.AutopilotConfig) // Wait for a contract to form. contractsFormed := cluster.WaitForContracts() @@ -2300,8 +2301,8 @@ func TestBusRecordedMetrics(t *testing.T) { for _, m := range csMetrics { if m.Contracts != 1 { t.Fatalf("expected 1 contract, got %v", m.Contracts) - } else if m.Name != testContractSet { - t.Fatalf("expected contract set %v, got %v", testContractSet, m.Name) + } else if m.Name != test.ContractSet { + t.Fatalf("expected contract set %v, got %v", test.ContractSet, m.Name) } else if m.Timestamp.Std().Before(startTime) { t.Fatalf("expected time to be after start time %v, got %v", startTime, m.Timestamp.Std()) } @@ -2317,8 +2318,8 @@ func TestBusRecordedMetrics(t *testing.T) { t.Fatalf("expected added churn, got %v", m.Direction) } else if m.ContractID == (types.FileContractID{}) { t.Fatal("expected non-zero FCID") - } else if m.Name != testContractSet { - t.Fatalf("expected contract set %v, got %v", testContractSet, m.Name) + } else if m.Name != test.ContractSet { + t.Fatalf("expected contract set %v, got %v", test.ContractSet, m.Name) } else if m.Timestamp.Std().Before(startTime) { t.Fatalf("expected time to be after start time %v, got %v", startTime, m.Timestamp.Std()) } @@ -2377,14 +2378,14 @@ func TestMultipartUploadWrappedByPartialSlabs(t *testing.T) { } cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, uploadPacking: true, }) defer cluster.Shutdown() defer cluster.Shutdown() b := cluster.Bus w := cluster.Worker - slabSize := testRedundancySettings.SlabSizeNoRedundancy() + slabSize := test.RedundancySettings.SlabSizeNoRedundancy() tt := cluster.tt // start a new multipart upload. We upload the parts in reverse order diff --git a/internal/test/e2e/gouging_test.go b/internal/test/e2e/gouging_test.go index 423733ab6..60e2f9b5e 100644 --- a/internal/test/e2e/gouging_test.go +++ b/internal/test/e2e/gouging_test.go @@ -10,6 +10,7 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/test" "go.uber.org/zap/zapcore" "lukechampine.com/frand" ) @@ -21,12 +22,12 @@ func TestGouging(t *testing.T) { // create a new test cluster cluster := newTestCluster(t, testClusterOptions{ - hosts: int(testAutopilotConfig.Contracts.Amount), + hosts: int(test.AutopilotConfig.Contracts.Amount), logger: newTestLoggerCustom(zapcore.ErrorLevel), }) defer cluster.Shutdown() - cfg := testAutopilotConfig.Contracts + cfg := test.AutopilotConfig.Contracts b := cluster.Bus w := cluster.Worker tt := cluster.tt diff --git a/internal/test/e2e/metadata_test.go b/internal/test/e2e/metadata_test.go index 47fb6a2b1..b71078eef 100644 --- a/internal/test/e2e/metadata_test.go +++ b/internal/test/e2e/metadata_test.go @@ -7,6 +7,7 @@ import ( "testing" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/test" "go.uber.org/zap" ) @@ -17,7 +18,7 @@ func TestObjectMetadata(t *testing.T) { // create cluster cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, logger: zap.NewNop(), }) defer cluster.Shutdown() diff --git a/internal/test/e2e/metrics_test.go b/internal/test/e2e/metrics_test.go index ed432c3c2..aaa139102 100644 --- a/internal/test/e2e/metrics_test.go +++ b/internal/test/e2e/metrics_test.go @@ -10,6 +10,7 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/test" "lukechampine.com/frand" ) @@ -22,12 +23,12 @@ func TestMetrics(t *testing.T) { start := time.Now() // enable pruning - apCfg := testAutopilotConfig + apCfg := test.AutopilotConfig apCfg.Contracts.Prune = true // create a test cluster cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, autopilotSettings: &apCfg, }) defer cluster.Shutdown() diff --git a/internal/test/e2e/migrations_test.go b/internal/test/e2e/migrations_test.go index 66325cc15..91bcc20b7 100644 --- a/internal/test/e2e/migrations_test.go +++ b/internal/test/e2e/migrations_test.go @@ -10,6 +10,7 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/test" "lukechampine.com/frand" ) @@ -19,13 +20,13 @@ func TestMigrations(t *testing.T) { } // create a new test cluster - cfg := testAutopilotConfig - cfg.Contracts.Amount = uint64(testRedundancySettings.TotalShards) + 1 + cfg := test.AutopilotConfig + cfg.Contracts.Amount = uint64(test.RedundancySettings.TotalShards) + 1 cluster := newTestCluster(t, testClusterOptions{ // configure the cluster to use 1 more host than the total shards in the // redundancy settings. autopilotSettings: &cfg, - hosts: int(testRedundancySettings.TotalShards) + 1, + hosts: int(test.RedundancySettings.TotalShards) + 1, }) defer cluster.Shutdown() @@ -60,8 +61,8 @@ func TestMigrations(t *testing.T) { // assert amount of hosts used used := usedHosts(path) - if len(used) != testRedundancySettings.TotalShards { - t.Fatal("unexpected amount of hosts used", len(used), testRedundancySettings.TotalShards) + if len(used) != test.RedundancySettings.TotalShards { + t.Fatal("unexpected amount of hosts used", len(used), test.RedundancySettings.TotalShards) } // select one host to remove diff --git a/internal/test/e2e/pruning_test.go b/internal/test/e2e/pruning_test.go index bd1bb822d..2a9f91000 100644 --- a/internal/test/e2e/pruning_test.go +++ b/internal/test/e2e/pruning_test.go @@ -13,6 +13,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" "go.sia.tech/renterd/hostdb" + "go.sia.tech/renterd/internal/test" ) func TestHostPruning(t *testing.T) { @@ -136,8 +137,8 @@ func TestSectorPruning(t *testing.T) { } // convenience variables - cfg := testAutopilotConfig - rs := testRedundancySettings + cfg := test.AutopilotConfig + rs := test.RedundancySettings w := cluster.Worker b := cluster.Bus tt := cluster.tt diff --git a/internal/test/e2e/s3_test.go b/internal/test/e2e/s3_test.go index 92b37af04..b25e11871 100644 --- a/internal/test/e2e/s3_test.go +++ b/internal/test/e2e/s3_test.go @@ -15,6 +15,7 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/gofakes3" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/test" "go.uber.org/zap" "lukechampine.com/frand" ) @@ -30,7 +31,7 @@ func TestS3Basic(t *testing.T) { start := time.Now() cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, }) defer cluster.Shutdown() @@ -176,7 +177,7 @@ func TestS3ObjectMetadata(t *testing.T) { // create cluster opts := testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, logger: zap.NewNop(), } cluster := newTestCluster(t, opts) @@ -288,7 +289,7 @@ func TestS3Authentication(t *testing.T) { // Create client with credentials and try again.. s3Authenticated, err := minio.NewCore(url, &minio.Options{ - Creds: testS3Credentials, + Creds: test.S3Credentials, }) tt.OK(err) @@ -328,7 +329,7 @@ func TestS3Authentication(t *testing.T) { func TestS3List(t *testing.T) { cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, uploadPacking: true, }) defer cluster.Shutdown() @@ -463,7 +464,7 @@ func TestS3MultipartUploads(t *testing.T) { } cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, uploadPacking: true, }) defer cluster.Shutdown() @@ -594,7 +595,7 @@ func TestS3MultipartPruneSlabs(t *testing.T) { } cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, uploadPacking: true, }) defer cluster.Shutdown() @@ -623,7 +624,7 @@ func TestS3MultipartPruneSlabs(t *testing.T) { // Upload 1 regular object. It will share the same packed slab, cause the // packed slab to be complete and start a new one. - data = frand.Bytes(testRedundancySettings.MinShards*rhpv2.SectorSize - 1) + data = frand.Bytes(test.RedundancySettings.MinShards*rhpv2.SectorSize - 1) tt.OKAll(s3.PutObject(context.Background(), bucket, "bar", bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{})) // Block until the buffer is uploaded. @@ -648,7 +649,7 @@ func TestS3SpecialChars(t *testing.T) { } cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, uploadPacking: true, }) defer cluster.Shutdown() diff --git a/internal/test/e2e/uploads_test.go b/internal/test/e2e/uploads_test.go index 96daae8c8..3f83fd7e4 100644 --- a/internal/test/e2e/uploads_test.go +++ b/internal/test/e2e/uploads_test.go @@ -10,6 +10,7 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/test" "lukechampine.com/frand" ) @@ -49,12 +50,12 @@ func TestUploadingSectorsCache(t *testing.T) { } cluster := newTestCluster(t, testClusterOptions{ - hosts: testRedundancySettings.TotalShards, + hosts: test.RedundancySettings.TotalShards, }) defer cluster.Shutdown() w := cluster.Worker b := cluster.Bus - rs := testRedundancySettings + rs := test.RedundancySettings tt := cluster.tt // generate some random data