From 4dd89b0481faff566dd61492df973d26b42a1222 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 16 Apr 2024 14:28:16 +0200 Subject: [PATCH 1/2] api;stores;contractor: add stored data to api.Host type and make use of it in the contractor --- api/host.go | 1 + autopilot/contractor/contractor.go | 14 +++++------ autopilot/contractor/evaluate.go | 4 ++-- autopilot/contractor/hostfilter.go | 4 ++-- autopilot/contractor/hostscore.go | 4 ++-- autopilot/contractor/hostscore_test.go | 18 +++++++------- internal/test/e2e/cluster_test.go | 13 +++++++++- stores/hostdb.go | 33 +++++++++++++++++++------- 8 files changed, 59 insertions(+), 32 deletions(-) diff --git a/api/host.go b/api/host.go index e4d472495..4ad1f87a1 100644 --- a/api/host.go +++ b/api/host.go @@ -158,6 +158,7 @@ type ( Scanned bool `json:"scanned"` Blocked bool `json:"blocked"` Checks map[string]HostCheck `json:"checks"` + StoredData uint64 `json:"storedData"` } HostAddress struct { diff --git a/autopilot/contractor/contractor.go b/autopilot/contractor/contractor.go index 00bbef98b..627ac92d1 100644 --- a/autopilot/contractor/contractor.go +++ b/autopilot/contractor/contractor.go @@ -284,10 +284,8 @@ func (c *Contractor) performContractMaintenance(ctx *mCtx, w Worker) (bool, erro // compile map of stored data per host contractData := make(map[types.FileContractID]uint64) - hostData := make(map[types.PublicKey]uint64) for _, c := range contracts { contractData[c.ID] = c.FileSize() - hostData[c.HostKey] += c.FileSize() } // fetch all hosts @@ -310,7 +308,7 @@ func (c *Contractor) performContractMaintenance(ctx *mCtx, w Worker) (bool, erro } // fetch candidate hosts - candidates, unusableHosts, err := c.candidateHosts(mCtx, hosts, usedHosts, hostData, minValidScore) // avoid 0 score hosts + candidates, unusableHosts, err := c.candidateHosts(mCtx, hosts, usedHosts, minValidScore) // avoid 0 score hosts if err != nil { return false, err } @@ -324,7 +322,7 @@ func (c *Contractor) performContractMaintenance(ctx *mCtx, w Worker) (bool, erro } // run host checks - checks, err := c.runHostChecks(mCtx, hosts, hostData, minScore) + checks, err := c.runHostChecks(mCtx, hosts, minScore) if err != nil { return false, fmt.Errorf("failed to run host checks, err: %v", err) } @@ -742,7 +740,7 @@ LOOP: return toKeep, toArchive, toStopUsing, toRefresh, toRenew } -func (c *Contractor) runHostChecks(ctx *mCtx, hosts []api.Host, hostData map[types.PublicKey]uint64, minScore float64) (map[types.PublicKey]*api.HostCheck, error) { +func (c *Contractor) runHostChecks(ctx *mCtx, hosts []api.Host, minScore float64) (map[types.PublicKey]*api.HostCheck, error) { // fetch consensus state cs, err := c.bus.ConsensusState(ctx) if err != nil { @@ -756,7 +754,7 @@ func (c *Contractor) runHostChecks(ctx *mCtx, hosts []api.Host, hostData map[typ checks := make(map[types.PublicKey]*api.HostCheck) for _, h := range hosts { h.PriceTable.HostBlockHeight = cs.BlockHeight // ignore HostBlockHeight - checks[h.PublicKey] = checkHost(ctx.AutopilotConfig(), ctx.state.RS, gc, h, minScore, hostData[h.PublicKey]) + checks[h.PublicKey] = checkHost(ctx.AutopilotConfig(), ctx.state.RS, gc, h, minScore) } return checks, nil } @@ -1220,7 +1218,7 @@ func (c *Contractor) calculateMinScore(candidates []scoredHost, numContracts uin return minScore } -func (c *Contractor) candidateHosts(ctx *mCtx, hosts []api.Host, usedHosts map[types.PublicKey]struct{}, storedData map[types.PublicKey]uint64, minScore float64) ([]scoredHost, unusableHostsBreakdown, error) { +func (c *Contractor) candidateHosts(ctx *mCtx, hosts []api.Host, usedHosts map[types.PublicKey]struct{}, minScore float64) ([]scoredHost, unusableHostsBreakdown, error) { start := time.Now() // fetch consensus state @@ -1273,7 +1271,7 @@ func (c *Contractor) candidateHosts(ctx *mCtx, hosts []api.Host, usedHosts map[t // NOTE: ignore the pricetable's HostBlockHeight by setting it to our // own blockheight h.PriceTable.HostBlockHeight = cs.BlockHeight - hc := checkHost(ctx.AutopilotConfig(), ctx.state.RS, gc, h, minScore, storedData[h.PublicKey]) + hc := checkHost(ctx.AutopilotConfig(), ctx.state.RS, gc, h, minScore) if hc.Usability.IsUsable() { candidates = append(candidates, scoredHost{h, hc.Score.Score()}) continue diff --git a/autopilot/contractor/evaluate.go b/autopilot/contractor/evaluate.go index cc964b3d4..685cb4b70 100644 --- a/autopilot/contractor/evaluate.go +++ b/autopilot/contractor/evaluate.go @@ -9,7 +9,7 @@ import ( func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.Host) (usables uint64) { gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow) for _, host := range hosts { - hc := checkHost(cfg, rs, gc, host, minValidScore, 0) + hc := checkHost(cfg, rs, gc, host, minValidScore) if hc.Usability.IsUsable() { usables++ } @@ -25,7 +25,7 @@ func EvaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Cu resp.Hosts = uint64(len(hosts)) for _, host := range hosts { - hc := checkHost(cfg, rs, gc, host, 0, 0) + hc := checkHost(cfg, rs, gc, host, 0) if hc.Usability.IsUsable() { resp.Usable++ continue diff --git a/autopilot/contractor/hostfilter.go b/autopilot/contractor/hostfilter.go index bfc11b903..dc95b1386 100644 --- a/autopilot/contractor/hostfilter.go +++ b/autopilot/contractor/hostfilter.go @@ -236,7 +236,7 @@ func isUpForRenewal(cfg api.AutopilotConfig, r types.FileContractRevision, block } // checkHost performs a series of checks on the host. -func checkHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.GougingChecker, h api.Host, minScore float64, storedData uint64) *api.HostCheck { +func checkHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.GougingChecker, h api.Host, minScore float64) *api.HostCheck { if rs.Validate() != nil { panic("invalid redundancy settings were supplied - developer error") } @@ -278,7 +278,7 @@ func checkHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.Gou // not gouging, this because the core package does not have overflow // checks in its cost calculations needed to calculate the period // cost - sb = hostScore(cfg, h, storedData, rs.Redundancy()) + sb = hostScore(cfg, h, rs.Redundancy()) if sb.Score() < minScore { ub.LowScore = true } diff --git a/autopilot/contractor/hostscore.go b/autopilot/contractor/hostscore.go index 3a05a947a..51d8275fc 100644 --- a/autopilot/contractor/hostscore.go +++ b/autopilot/contractor/hostscore.go @@ -22,7 +22,7 @@ const ( minValidScore = math.SmallestNonzeroFloat64 ) -func hostScore(cfg api.AutopilotConfig, h api.Host, storedData uint64, expectedRedundancy float64) api.HostScoreBreakdown { +func hostScore(cfg api.AutopilotConfig, h api.Host, expectedRedundancy float64) api.HostScoreBreakdown { cCfg := cfg.Contracts // idealDataPerHost is the amount of data that we would have to put on each // host assuming that our storage requirements were spread evenly across @@ -44,7 +44,7 @@ func hostScore(cfg api.AutopilotConfig, h api.Host, storedData uint64, expectedR Collateral: collateralScore(cCfg, h.PriceTable.HostPriceTable, uint64(allocationPerHost)), Interactions: interactionScore(h), Prices: priceAdjustmentScore(hostPeriodCost, cCfg), - StorageRemaining: storageRemainingScore(h.Settings, storedData, allocationPerHost), + StorageRemaining: storageRemainingScore(h.Settings, h.StoredData, allocationPerHost), Uptime: uptimeScore(h), Version: versionScore(h.Settings, cfg.Hosts.MinProtocolVersion), } diff --git a/autopilot/contractor/hostscore_test.go b/autopilot/contractor/hostscore_test.go index 84f964692..ae1b7668e 100644 --- a/autopilot/contractor/hostscore_test.go +++ b/autopilot/contractor/hostscore_test.go @@ -42,13 +42,13 @@ func TestHostScore(t *testing.T) { // assert both hosts score equal redundancy := 3.0 - if hostScore(cfg, h1, 0, redundancy) != hostScore(cfg, h2, 0, redundancy) { + if hostScore(cfg, h1, redundancy) != hostScore(cfg, h2, redundancy) { t.Fatal("unexpected") } // assert age affects the score h1.KnownSince = time.Now().Add(-1 * day) - if hostScore(cfg, h1, 0, redundancy).Score() <= hostScore(cfg, h2, 0, redundancy).Score() { + if hostScore(cfg, h1, redundancy).Score() <= hostScore(cfg, h2, redundancy).Score() { t.Fatal("unexpected") } @@ -57,21 +57,21 @@ func TestHostScore(t *testing.T) { settings.Collateral = settings.Collateral.Div64(2) settings.MaxCollateral = settings.MaxCollateral.Div64(2) h1 = newHost(settings) // reset - if hostScore(cfg, h1, 0, redundancy).Score() <= hostScore(cfg, h2, 0, redundancy).Score() { + if hostScore(cfg, h1, redundancy).Score() <= hostScore(cfg, h2, redundancy).Score() { t.Fatal("unexpected") } // assert interactions affect the score h1 = newHost(test.NewHostSettings()) // reset h1.Interactions.SuccessfulInteractions++ - if hostScore(cfg, h1, 0, redundancy).Score() <= hostScore(cfg, h2, 0, redundancy).Score() { + if hostScore(cfg, h1, redundancy).Score() <= hostScore(cfg, h2, redundancy).Score() { t.Fatal("unexpected") } // assert uptime affects the score h2 = newHost(test.NewHostSettings()) // reset h2.Interactions.SecondToLastScanSuccess = false - if hostScore(cfg, h1, 0, redundancy).Score() <= hostScore(cfg, h2, 0, redundancy).Score() || ageScore(h1) != ageScore(h2) { + if hostScore(cfg, h1, redundancy).Score() <= hostScore(cfg, h2, redundancy).Score() || ageScore(h1) != ageScore(h2) { t.Fatal("unexpected") } @@ -79,28 +79,28 @@ func TestHostScore(t *testing.T) { h2Settings := test.NewHostSettings() h2Settings.Version = "1.5.6" // lower h2 = newHost(h2Settings) // reset - if hostScore(cfg, h1, 0, redundancy).Score() <= hostScore(cfg, h2, 0, redundancy).Score() { + if hostScore(cfg, h1, redundancy).Score() <= hostScore(cfg, h2, redundancy).Score() { t.Fatal("unexpected") } // asseret remaining storage affects the score. h1 = newHost(test.NewHostSettings()) // reset h2.Settings.RemainingStorage = 100 - if hostScore(cfg, h1, 0, redundancy).Score() <= hostScore(cfg, h2, 0, redundancy).Score() { + if hostScore(cfg, h1, redundancy).Score() <= hostScore(cfg, h2, redundancy).Score() { t.Fatal("unexpected") } // assert MaxCollateral affects the score. h2 = newHost(test.NewHostSettings()) // reset h2.PriceTable.MaxCollateral = types.ZeroCurrency - if hostScore(cfg, h1, 0, redundancy).Score() <= hostScore(cfg, h2, 0, redundancy).Score() { + if hostScore(cfg, h1, redundancy).Score() <= hostScore(cfg, h2, redundancy).Score() { t.Fatal("unexpected") } // assert price affects the score. h2 = newHost(test.NewHostSettings()) // reset h2.PriceTable.WriteBaseCost = types.Siacoins(1) - if hostScore(cfg, h1, 0, redundancy).Score() <= hostScore(cfg, h2, 0, redundancy).Score() { + if hostScore(cfg, h1, redundancy).Score() <= hostScore(cfg, h2, redundancy).Score() { t.Fatal("unexpected") } } diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 55db48c19..a91858aef 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -612,7 +612,7 @@ func TestUploadDownloadBasic(t *testing.T) { // mine a block to get the revisions mined. cluster.MineBlocks(1) - // check the revision height was updated. + // check the revision height and size were updated. tt.Retry(100, 100*time.Millisecond, func() error { // fetch the contracts. contracts, err := cluster.Bus.Contracts(context.Background(), api.ContractsOpts{}) @@ -623,10 +623,21 @@ func TestUploadDownloadBasic(t *testing.T) { for _, c := range contracts { if c.RevisionHeight == 0 { return errors.New("revision height should be > 0") + } else if c.Size != rhpv2.SectorSize { + return fmt.Errorf("size should be %v, got %v", rhpv2.SectorSize, c.Size) } } return nil }) + + // Check that stored data on hosts was updated + hosts, err := cluster.Bus.Hosts(context.Background(), api.GetHostsOptions{}) + tt.OK(err) + for _, host := range hosts { + if host.StoredData != rhpv2.SectorSize { + t.Fatalf("stored data should be %v, got %v", rhpv2.SectorSize, host.StoredData) + } + } } // TestUploadDownloadExtended is an integration test that verifies objects can diff --git a/stores/hostdb.go b/stores/hostdb.go index fa93f85b9..0aa3ab0b2 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -255,7 +255,7 @@ func (dbAllowlistEntry) TableName() string { return "host_allowlist_entries" } func (dbBlocklistEntry) TableName() string { return "host_blocklist_entries" } // convert converts a host into a api.HostInfo -func (h dbHost) convert(blocked bool) api.Host { +func (h dbHost) convert(blocked bool, storedData uint64) api.Host { var lastScan time.Time if h.LastScan > 0 { lastScan = time.Unix(0, h.LastScan) @@ -283,11 +283,12 @@ func (h dbHost) convert(blocked bool) api.Host { HostPriceTable: h.PriceTable.convert(), Expiry: h.PriceTableExpiry.Time, }, - PublicKey: types.PublicKey(h.PublicKey), - Scanned: h.Scanned, - Settings: rhpv2.HostSettings(h.Settings), - Blocked: blocked, - Checks: checks, + PublicKey: types.PublicKey(h.PublicKey), + Scanned: h.Scanned, + Settings: rhpv2.HostSettings(h.Settings), + Blocked: blocked, + Checks: checks, + StoredData: storedData, } } @@ -571,9 +572,25 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, autopilotID, filterMode, us Preload("Blocklist") } + // fetch stored data for each host + var storedData []struct { + HostID uint + StoredData uint64 + } + err := ss.db.Raw("SELECT host_id, SUM(size) as StoredData FROM contracts GROUP BY host_id"). + Scan(&storedData). + Error + if err != nil { + return nil, fmt.Errorf("failed to fetch stored data: %w", err) + } + storedDataMap := make(map[uint]uint64) + for _, host := range storedData { + storedDataMap[host.HostID] = host.StoredData + } + var hosts []api.Host var fullHosts []dbHost - err := query. + err = query. Offset(offset). Limit(limit). FindInBatches(&fullHosts, hostRetrievalBatchSize, func(tx *gorm.DB, batch int) error { @@ -584,7 +601,7 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, autopilotID, filterMode, us } else { blocked = filterMode == api.HostFilterModeBlocked } - hosts = append(hosts, fh.convert(blocked)) + hosts = append(hosts, fh.convert(blocked, storedDataMap[fh.ID])) } return nil }). From 1bea53efff32ba3e80435e5bb2317b07779b9728 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 17 Apr 2024 13:51:38 +0200 Subject: [PATCH 2/2] contractor: address comments --- autopilot/contractor/contractor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autopilot/contractor/contractor.go b/autopilot/contractor/contractor.go index 627ac92d1..a2a8e81e1 100644 --- a/autopilot/contractor/contractor.go +++ b/autopilot/contractor/contractor.go @@ -282,7 +282,7 @@ func (c *Contractor) performContractMaintenance(ctx *mCtx, w Worker) (bool, erro usedHosts[contract.HostKey] = struct{}{} } - // compile map of stored data per host + // compile map of stored data per contract contractData := make(map[types.FileContractID]uint64) for _, c := range contracts { contractData[c.ID] = c.FileSize()