diff --git a/.github/workflows/project-add.yml b/.github/workflows/project-add.yml index c61a17b0d..8b931016b 100644 --- a/.github/workflows/project-add.yml +++ b/.github/workflows/project-add.yml @@ -12,4 +12,3 @@ jobs: add-to-project: uses: SiaFoundation/workflows/.github/workflows/project-add.yml@master secrets: inherit - diff --git a/api/autopilot.go b/api/autopilot.go index 22598b28c..23425aacf 100644 --- a/api/autopilot.go +++ b/api/autopilot.go @@ -4,7 +4,6 @@ import ( "errors" "go.sia.tech/core/types" - "go.sia.tech/renterd/hostdb" ) const ( @@ -119,21 +118,6 @@ type ( } Recommendation *ConfigRecommendation `json:"recommendation,omitempty"` } - - // HostHandlerResponse is the response type for the /host/:hostkey endpoint. - HostHandlerResponse struct { - Host hostdb.Host `json:"host"` - Checks *HostHandlerResponseChecks `json:"checks,omitempty"` - } - - HostHandlerResponseChecks struct { - Gouging bool `json:"gouging"` - GougingBreakdown HostGougingBreakdown `json:"gougingBreakdown"` - Score float64 `json:"score"` - ScoreBreakdown HostScoreBreakdown `json:"scoreBreakdown"` - Usable bool `json:"usable"` - UnusableReasons []string `json:"unusableReasons"` - } ) func (c AutopilotConfig) Validate() error { diff --git a/api/host.go b/api/host.go index 5221fb20c..e4d472495 100644 --- a/api/host.go +++ b/api/host.go @@ -5,9 +5,11 @@ import ( "fmt" "net/url" "strings" + "time" + rhpv2 "go.sia.tech/core/rhp/v2" + rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" - "go.sia.tech/renterd/hostdb" ) const ( @@ -26,15 +28,27 @@ var ( ErrHostNotFound = errors.New("host doesn't exist in hostdb") ) +var ( + ErrUsabilityHostBlocked = errors.New("host is blocked") + ErrUsabilityHostNotFound = errors.New("host not found") + ErrUsabilityHostOffline = errors.New("host is offline") + ErrUsabilityHostLowScore = errors.New("host's score is below minimum") + ErrUsabilityHostRedundantIP = errors.New("host has redundant IP") + ErrUsabilityHostPriceGouging = errors.New("host is price gouging") + ErrUsabilityHostNotAcceptingContracts = errors.New("host is not accepting contracts") + ErrUsabilityHostNotCompletingScan = errors.New("host is not completing scan") + ErrUsabilityHostNotAnnounced = errors.New("host is not announced") +) + type ( // HostsScanRequest is the request type for the /hosts/scans endpoint. HostsScanRequest struct { - Scans []hostdb.HostScan `json:"scans"` + Scans []HostScan `json:"scans"` } // HostsPriceTablesRequest is the request type for the /hosts/pricetables endpoint. HostsPriceTablesRequest struct { - PriceTableUpdates []hostdb.PriceTableUpdate `json:"priceTableUpdates"` + PriceTableUpdates []HostPriceTableUpdate `json:"priceTableUpdates"` } // HostsRemoveRequest is the request type for the /hosts/remove endpoint. @@ -48,11 +62,28 @@ type ( SearchHostsRequest struct { Offset int `json:"offset"` Limit int `json:"limit"` + AutopilotID string `json:"autopilotID"` FilterMode string `json:"filterMode"` UsabilityMode string `json:"usabilityMode"` AddressContains string `json:"addressContains"` KeyIn []types.PublicKey `json:"keyIn"` } + + // HostResponse is the response type for the GET + // /api/autopilot/host/:hostkey endpoint. + HostResponse struct { + Host Host `json:"host"` + Checks *HostChecks `json:"checks,omitempty"` + } + + HostChecks struct { + Gouging bool `json:"gouging"` + GougingBreakdown HostGougingBreakdown `json:"gougingBreakdown"` + Score float64 `json:"score"` + ScoreBreakdown HostScoreBreakdown `json:"scoreBreakdown"` + Usable bool `json:"usable"` + UnusableReasons []string `json:"unusableReasons,omitempty"` + } ) type ( @@ -84,8 +115,10 @@ type ( } SearchHostOptions struct { + AutopilotID string AddressContains string FilterMode string + UsabilityMode string KeyIn []types.PublicKey Limit int Offset int @@ -115,9 +148,54 @@ func (opts HostsForScanningOptions) Apply(values url.Values) { type ( Host struct { - hostdb.Host - Blocked bool `json:"blocked"` - Checks map[string]HostCheck `json:"checks"` + KnownSince time.Time `json:"knownSince"` + LastAnnouncement time.Time `json:"lastAnnouncement"` + PublicKey types.PublicKey `json:"publicKey"` + NetAddress string `json:"netAddress"` + PriceTable HostPriceTable `json:"priceTable"` + Settings rhpv2.HostSettings `json:"settings"` + Interactions HostInteractions `json:"interactions"` + Scanned bool `json:"scanned"` + Blocked bool `json:"blocked"` + Checks map[string]HostCheck `json:"checks"` + } + + HostAddress struct { + PublicKey types.PublicKey `json:"publicKey"` + NetAddress string `json:"netAddress"` + } + + HostInteractions struct { + TotalScans uint64 `json:"totalScans"` + LastScan time.Time `json:"lastScan"` + LastScanSuccess bool `json:"lastScanSuccess"` + LostSectors uint64 `json:"lostSectors"` + SecondToLastScanSuccess bool `json:"secondToLastScanSuccess"` + Uptime time.Duration `json:"uptime"` + Downtime time.Duration `json:"downtime"` + + SuccessfulInteractions float64 `json:"successfulInteractions"` + FailedInteractions float64 `json:"failedInteractions"` + } + + HostScan struct { + HostKey types.PublicKey `json:"hostKey"` + Success bool + Timestamp time.Time + Settings rhpv2.HostSettings + PriceTable rhpv3.HostPriceTable + } + + HostPriceTable struct { + rhpv3.HostPriceTable + Expiry time.Time `json:"expiry"` + } + + HostPriceTableUpdate struct { + HostKey types.PublicKey `json:"hostKey"` + Success bool + Timestamp time.Time + PriceTable HostPriceTable } HostCheck struct { @@ -156,6 +234,21 @@ type ( } ) +// IsAnnounced returns whether the host has been announced. +func (h Host) IsAnnounced() bool { + return !h.LastAnnouncement.IsZero() +} + +// IsOnline returns whether a host is considered online. +func (h Host) IsOnline() bool { + if h.Interactions.TotalScans == 0 { + return false + } else if h.Interactions.TotalScans == 1 { + return h.Interactions.LastScanSuccess + } + return h.Interactions.LastScanSuccess || h.Interactions.SecondToLastScanSuccess +} + func (sb HostScoreBreakdown) String() string { return fmt.Sprintf("Age: %v, Col: %v, Int: %v, SR: %v, UT: %v, V: %v, Pr: %v", sb.Age, sb.Collateral, sb.Interactions, sb.StorageRemaining, sb.Uptime, sb.Version, sb.Prices) } @@ -194,3 +287,36 @@ func (hgb HostGougingBreakdown) String() string { func (sb HostScoreBreakdown) Score() float64 { return sb.Age * sb.Collateral * sb.Interactions * sb.StorageRemaining * sb.Uptime * sb.Version * sb.Prices } + +func (ub HostUsabilityBreakdown) IsUsable() bool { + return !ub.Blocked && !ub.Offline && !ub.LowScore && !ub.RedundantIP && !ub.Gouging && !ub.NotAcceptingContracts && !ub.NotAnnounced && !ub.NotCompletingScan +} + +func (ub HostUsabilityBreakdown) UnusableReasons() []string { + var reasons []string + if ub.Blocked { + reasons = append(reasons, ErrUsabilityHostBlocked.Error()) + } + if ub.Offline { + reasons = append(reasons, ErrUsabilityHostOffline.Error()) + } + if ub.LowScore { + reasons = append(reasons, ErrUsabilityHostLowScore.Error()) + } + if ub.RedundantIP { + reasons = append(reasons, ErrUsabilityHostRedundantIP.Error()) + } + if ub.Gouging { + reasons = append(reasons, ErrUsabilityHostPriceGouging.Error()) + } + if ub.NotAcceptingContracts { + reasons = append(reasons, ErrUsabilityHostNotAcceptingContracts.Error()) + } + if ub.NotAnnounced { + reasons = append(reasons, ErrUsabilityHostNotAnnounced.Error()) + } + if ub.NotCompletingScan { + reasons = append(reasons, ErrUsabilityHostNotCompletingScan.Error()) + } + return reasons +} diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 57ec72967..4e3023a2f 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -17,7 +17,6 @@ import ( "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" "go.sia.tech/renterd/build" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/object" "go.sia.tech/renterd/wallet" @@ -54,9 +53,10 @@ type Bus interface { // hostdb Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) - HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error) + HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]api.HostAddress, error) RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.Host, error) + UpdateHostCheck(ctx context.Context, autopilotID string, hostKey types.PublicKey, hostCheck api.HostCheck) error // metrics RecordContractSetChurnMetric(ctx context.Context, metrics ...api.ContractSetChurnMetric) error @@ -682,16 +682,85 @@ func (ap *Autopilot) triggerHandlerPOST(jc jape.Context) { } func (ap *Autopilot) hostHandlerGET(jc jape.Context) { - var hostKey types.PublicKey - if jc.DecodeParam("hostKey", &hostKey) != nil { + var hk types.PublicKey + if jc.DecodeParam("hostKey", &hk) != nil { return } - host, err := ap.c.HostInfo(jc.Request.Context(), hostKey) + // TODO: remove on next major release + if jc.Check("failed to get host", compatV105Host(jc.Request.Context(), ap.State(), ap.bus, hk)) != nil { + return + } + + hi, err := ap.bus.Host(jc.Request.Context(), hk) if jc.Check("failed to get host info", err) != nil { return } - jc.Encode(host) + + check, ok := hi.Checks[ap.id] + if ok { + jc.Encode(api.HostResponse{ + Host: hi, + Checks: &api.HostChecks{ + Gouging: check.Gouging.Gouging(), + GougingBreakdown: check.Gouging, + Score: check.Score.Score(), + ScoreBreakdown: check.Score, + Usable: check.Usability.IsUsable(), + UnusableReasons: check.Usability.UnusableReasons(), + }, + }) + return + } + + jc.Encode(api.HostResponse{Host: hi}) +} + +func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { + var req api.SearchHostsRequest + if jc.Decode(&req) != nil { + return + } else if req.AutopilotID != "" && req.AutopilotID != ap.id { + jc.Error(errors.New("invalid autopilot id"), http.StatusBadRequest) + return + } + + // TODO: remove on next major release + if jc.Check("failed to get host info", compatV105UsabilityFilterModeCheck(req.UsabilityMode)) != nil { + return + } + + hosts, err := ap.bus.SearchHosts(jc.Request.Context(), api.SearchHostOptions{ + AutopilotID: ap.id, + Offset: req.Offset, + Limit: req.Limit, + FilterMode: req.FilterMode, + UsabilityMode: req.UsabilityMode, + AddressContains: req.AddressContains, + KeyIn: req.KeyIn, + }) + if jc.Check("failed to get host info", err) != nil { + return + } + resps := make([]api.HostResponse, len(hosts)) + for i, host := range hosts { + if check, ok := host.Checks[ap.id]; ok { + resps[i] = api.HostResponse{ + Host: host, + Checks: &api.HostChecks{ + Gouging: check.Gouging.Gouging(), + GougingBreakdown: check.Gouging, + Score: check.Score.Score(), + ScoreBreakdown: check.Score, + Usable: check.Usability.IsUsable(), + UnusableReasons: check.Usability.UnusableReasons(), + }, + } + } else { + resps[i] = api.HostResponse{Host: host} + } + } + jc.Encode(resps) } func (ap *Autopilot) stateHandlerGET(jc jape.Context) { @@ -725,23 +794,11 @@ func (ap *Autopilot) stateHandlerGET(jc jape.Context) { }) } -func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { - var req api.SearchHostsRequest - if jc.Decode(&req) != nil { - return - } - hosts, err := ap.c.HostInfos(jc.Request.Context(), req.FilterMode, req.UsabilityMode, req.AddressContains, req.KeyIn, req.Offset, req.Limit) - if jc.Check("failed to get host info", err) != nil { - return - } - jc.Encode(hosts) -} - func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.Host) (usables uint64) { gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow) for _, host := range hosts { - usable, _ := isUsableHost(cfg, rs, gc, host, smallestValidScore, 0) - if usable { + hc := checkHost(cfg, rs, gc, host, smallestValidScore, 0) + if hc.Usability.IsUsable() { usables++ } } @@ -756,36 +813,33 @@ func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Cu resp.Hosts = uint64(len(hosts)) for _, host := range hosts { - usable, usableBreakdown := isUsableHost(cfg, rs, gc, host, 0, 0) - if usable { + hc := checkHost(cfg, rs, gc, host, 0, 0) + if hc.Usability.IsUsable() { resp.Usable++ continue } - if usableBreakdown.blocked > 0 { + if hc.Usability.Blocked { resp.Unusable.Blocked++ } - if usableBreakdown.notacceptingcontracts > 0 { + if hc.Usability.NotAcceptingContracts { resp.Unusable.NotAcceptingContracts++ } - if usableBreakdown.notcompletingscan > 0 { + if hc.Usability.NotCompletingScan { resp.Unusable.NotScanned++ } - if usableBreakdown.unknown > 0 { - resp.Unusable.Unknown++ - } - if usableBreakdown.gougingBreakdown.ContractErr != "" { + if hc.Gouging.ContractErr != "" { resp.Unusable.Gouging.Contract++ } - if usableBreakdown.gougingBreakdown.DownloadErr != "" { + if hc.Gouging.DownloadErr != "" { resp.Unusable.Gouging.Download++ } - if usableBreakdown.gougingBreakdown.GougingErr != "" { + if hc.Gouging.GougingErr != "" { resp.Unusable.Gouging.Gouging++ } - if usableBreakdown.gougingBreakdown.PruneErr != "" { + if hc.Gouging.PruneErr != "" { resp.Unusable.Gouging.Pruning++ } - if usableBreakdown.gougingBreakdown.UploadErr != "" { + if hc.Gouging.UploadErr != "" { resp.Unusable.Gouging.Upload++ } } @@ -904,3 +958,56 @@ func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg nSteps++ } } + +// compatV105Host performs some state checks and bus calls we no longer need but +// are necessary checks to make sure our API is consistent. This should be +// removed in the next major release. +func compatV105Host(ctx context.Context, s state, b Bus, hk types.PublicKey) error { + // state checks + if s.cfg.Contracts.Allowance.IsZero() { + return fmt.Errorf("can not score hosts because contracts allowance is zero") + } + if s.cfg.Contracts.Amount == 0 { + return fmt.Errorf("can not score hosts because contracts amount is zero") + } + if s.cfg.Contracts.Period == 0 { + return fmt.Errorf("can not score hosts because contract period is zero") + } + + // fetch host + _, err := b.Host(ctx, hk) + if err != nil { + return fmt.Errorf("failed to fetch requested host from bus: %w", err) + } + + // other checks + _, err = b.GougingSettings(ctx) + if err != nil { + return fmt.Errorf("failed to fetch gouging settings from bus: %w", err) + } + _, err = b.RedundancySettings(ctx) + if err != nil { + return fmt.Errorf("failed to fetch redundancy settings from bus: %w", err) + } + _, err = b.ConsensusState(ctx) + if err != nil { + return fmt.Errorf("failed to fetch consensus state from bus: %w", err) + } + _, err = b.RecommendedFee(ctx) + if err != nil { + return fmt.Errorf("failed to fetch recommended fee from bus: %w", err) + } + return nil +} + +func compatV105UsabilityFilterModeCheck(usabilityMode string) error { + switch usabilityMode { + case api.UsabilityFilterModeUsable: + case api.UsabilityFilterModeUnusable: + case api.UsabilityFilterModeAll: + case "": + default: + return fmt.Errorf("invalid usability mode: '%v', options are 'usable', 'unusable' or an empty string for no filter", usabilityMode) + } + return nil +} diff --git a/autopilot/autopilot_test.go b/autopilot/autopilot_test.go index a21b55c7b..a572c56fc 100644 --- a/autopilot/autopilot_test.go +++ b/autopilot/autopilot_test.go @@ -9,7 +9,6 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" ) func TestOptimiseGougingSetting(t *testing.T) { @@ -18,32 +17,30 @@ func TestOptimiseGougingSetting(t *testing.T) { for i := 0; i < 10; i++ { hosts = append(hosts, api.Host{ - Host: hostdb.Host{ - KnownSince: time.Unix(0, 0), - PriceTable: hostdb.HostPriceTable{ - HostPriceTable: rhpv3.HostPriceTable{ - CollateralCost: types.Siacoins(1), - MaxCollateral: types.Siacoins(1000), - }, + KnownSince: time.Unix(0, 0), + PriceTable: api.HostPriceTable{ + HostPriceTable: rhpv3.HostPriceTable{ + CollateralCost: types.Siacoins(1), + MaxCollateral: types.Siacoins(1000), }, - Settings: rhpv2.HostSettings{ - AcceptingContracts: true, - Collateral: types.Siacoins(1), - MaxCollateral: types.Siacoins(1000), - Version: "1.6.0", - }, - Interactions: hostdb.Interactions{ - Uptime: time.Hour * 1000, - LastScan: time.Now(), - LastScanSuccess: true, - SecondToLastScanSuccess: true, - TotalScans: 100, - }, - LastAnnouncement: time.Unix(0, 0), - Scanned: true, }, - Blocked: false, - Checks: nil, + Settings: rhpv2.HostSettings{ + AcceptingContracts: true, + Collateral: types.Siacoins(1), + MaxCollateral: types.Siacoins(1000), + Version: "1.6.0", + }, + Interactions: api.HostInteractions{ + Uptime: time.Hour * 1000, + LastScan: time.Now(), + LastScanSuccess: true, + SecondToLastScanSuccess: true, + TotalScans: 100, + }, + LastAnnouncement: time.Unix(0, 0), + Scanned: true, + Blocked: false, + Checks: nil, }) } diff --git a/autopilot/client.go b/autopilot/client.go index 01d0a1632..010c1f037 100644 --- a/autopilot/client.go +++ b/autopilot/client.go @@ -34,13 +34,13 @@ func (c *Client) UpdateConfig(cfg api.AutopilotConfig) error { } // HostInfo returns information about the host with given host key. -func (c *Client) HostInfo(hostKey types.PublicKey) (resp api.HostHandlerResponse, err error) { +func (c *Client) HostInfo(hostKey types.PublicKey) (resp api.HostResponse, err error) { err = c.c.GET(fmt.Sprintf("/host/%s", hostKey), &resp) return } // HostInfo returns information about all hosts. -func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostHandlerResponse, err error) { +func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostResponse, err error) { err = c.c.POST("/hosts", api.SearchHostsRequest{ Offset: offset, Limit: limit, diff --git a/autopilot/contractor.go b/autopilot/contractor.go index d18ad6496..b3dd76ac4 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -15,7 +15,6 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/wallet" "go.sia.tech/renterd/worker" @@ -106,19 +105,10 @@ type ( pruning bool pruningLastStart time.Time - - cachedHostInfo map[types.PublicKey]hostInfo - cachedDataStored map[types.PublicKey]uint64 - cachedMinScore float64 - } - - hostInfo struct { - Usable bool - UnusableResult unusableHostResult } scoredHost struct { - host hostdb.Host + host api.Host score float64 } @@ -261,8 +251,8 @@ func (c *contractor) performContractMaintenance(ctx context.Context, w Worker) ( hostData[c.HostKey] += c.FileSize() } - // fetch all hosts - hosts, err := c.ap.bus.SearchHosts(ctx, api.SearchHostOptions{Limit: -1, FilterMode: api.HostFilterModeAllowed}) + // fetch all hosts (this includes blocked hosts) + hosts, err := c.ap.bus.SearchHosts(ctx, api.SearchHostOptions{Limit: -1, FilterMode: api.HostFilterModeAll}) if err != nil { return false, err } @@ -294,40 +284,28 @@ func (c *contractor) performContractMaintenance(ctx context.Context, w Worker) ( c.logger.Warn("could not calculate min score, no hosts found") } + // run host checks + checks, err := c.runHostChecks(ctx, hosts, hostData, minScore) + if err != nil { + return false, fmt.Errorf("failed to run host checks, err: %v", err) + } + // fetch consensus state cs, err := c.ap.bus.ConsensusState(ctx) if err != nil { - return false, err + return false, fmt.Errorf("failed to fetch consensus state, err: %v", err) } - // create gouging checker - gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) + // run contract checks + updatedSet, toArchive, toStopUsing, toRefresh, toRenew := c.runContractChecks(ctx, contracts, isInCurrentSet, checks, cs.BlockHeight) - // prepare hosts for cache - hostInfos := make(map[types.PublicKey]hostInfo) - for _, h := range hosts { - // ignore the pricetable's HostBlockHeight by setting it to our own blockheight - h.PriceTable.HostBlockHeight = cs.BlockHeight - isUsable, unusableResult := isUsableHost(state.cfg, state.rs, gc, h, minScore, hostData[h.PublicKey]) - hostInfos[h.PublicKey] = hostInfo{ - Usable: isUsable, - UnusableResult: unusableResult, + // update host checks + for hk, check := range checks { + if err := c.ap.bus.UpdateHostCheck(ctx, c.ap.id, hk, *check); err != nil { + c.logger.Errorf("failed to update host check for host %v, err: %v", hk, err) } } - // update cache. - c.mu.Lock() - c.cachedHostInfo = hostInfos - c.cachedDataStored = hostData - c.cachedMinScore = minScore - c.mu.Unlock() - - // run checks - updatedSet, toArchive, toStopUsing, toRefresh, toRenew, err := c.runContractChecks(ctx, w, contracts, isInCurrentSet, minScore) - if err != nil { - return false, fmt.Errorf("failed to run contract checks, err: %v", err) - } - // archive contracts if len(toArchive) > 0 { c.logger.Infof("archiving %d contracts: %+v", len(toArchive), toArchive) @@ -672,7 +650,7 @@ func (c *contractor) performWalletMaintenance(ctx context.Context) error { return nil } -func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts []api.Contract, inCurrentSet map[types.FileContractID]struct{}, minScore float64) (toKeep []api.ContractMetadata, toArchive, toStopUsing map[types.FileContractID]string, toRefresh, toRenew []contractInfo, _ error) { +func (c *contractor) runContractChecks(ctx context.Context, contracts []api.Contract, inCurrentSet map[types.FileContractID]struct{}, hostChecks map[types.PublicKey]*api.HostCheck, bh uint64) (toKeep []api.ContractMetadata, toArchive, toStopUsing map[types.FileContractID]string, toRefresh, toRenew []contractInfo) { if c.ap.isStopped() { return } @@ -681,12 +659,6 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts // convenience variables state := c.ap.State() - // fetch consensus state - cs, err := c.ap.bus.ConsensusState(ctx) - if err != nil { - return nil, nil, nil, nil, nil, err - } - // create new IP filter ipFilter := c.newIPFilter() @@ -730,15 +702,16 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts // convenience variables fcid := contract.ID + hk := contract.HostKey // check if contract is ready to be archived. - if cs.BlockHeight > contract.EndHeight()-c.revisionSubmissionBuffer { + if bh > contract.EndHeight()-c.revisionSubmissionBuffer { toArchive[fcid] = errContractExpired.Error() } else if contract.Revision != nil && contract.Revision.RevisionNumber == math.MaxUint64 { toArchive[fcid] = errContractMaxRevisionNumber.Error() } else if contract.RevisionNumber == math.MaxUint64 { toArchive[fcid] = errContractMaxRevisionNumber.Error() - } else if contract.State == api.ContractStatePending && cs.BlockHeight-contract.StartHeight > contractConfirmationDeadline { + } else if contract.State == api.ContractStatePending && bh-contract.StartHeight > contractConfirmationDeadline { toArchive[fcid] = errContractNotConfirmed.Error() } if _, archived := toArchive[fcid]; archived { @@ -747,52 +720,34 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts } // fetch host from hostdb - hk := contract.HostKey host, err := c.ap.bus.Host(ctx, hk) if err != nil { - c.logger.Errorw(fmt.Sprintf("missing host, err: %v", err), "hk", hk) - toStopUsing[fcid] = errHostNotFound.Error() + c.logger.Warn(fmt.Sprintf("missing host, err: %v", err), "hk", hk) + toStopUsing[fcid] = api.ErrUsabilityHostNotFound.Error() notfound++ continue } - // if the host is blocked we ignore it, it might be unblocked later - if host.Blocked { - c.logger.Infow("unusable host", "hk", hk, "fcid", fcid, "reasons", errHostBlocked.Error()) - toStopUsing[fcid] = errHostBlocked.Error() + // fetch host checks + check, ok := hostChecks[hk] + if !ok { + // this is only possible due to developer error, if there is no + // check the host would have been missing, so we treat it the same + c.logger.Warnw("missing host check", "hk", hk) + toStopUsing[fcid] = api.ErrUsabilityHostNotFound.Error() continue } - // if the host doesn't have a valid pricetable, update it if we were - // able to obtain a revision - invalidPT := contract.Revision == nil - if contract.Revision != nil { - if err := refreshPriceTable(ctx, w, &host.Host); err != nil { - c.logger.Errorf("could not fetch price table for host %v: %v", host.PublicKey, err) - invalidPT = true - } - } - - // refresh the consensus state - if css, err := c.ap.bus.ConsensusState(ctx); err != nil { - c.logger.Errorf("could not fetch consensus state, err: %v", err) - } else { - cs = css + // if the host is blocked we ignore it, it might be unblocked later + if host.Blocked { + c.logger.Infow("unusable host", "hk", hk, "fcid", fcid, "reasons", api.ErrUsabilityHostBlocked.Error()) + toStopUsing[fcid] = api.ErrUsabilityHostBlocked.Error() + continue } - // use a new gouging checker for every contract - gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) - - // set the host's block height to ours to disable the height check in - // the gouging checks, in certain edge cases the renter might unsync and - // would therefor label all hosts as unusable and go on to create a - // whole new set of contracts with new hosts - host.PriceTable.HostBlockHeight = cs.BlockHeight - - // decide whether the host is still good - usable, unusableResult := isUsableHost(state.cfg, state.rs, gc, host, minScore, contract.FileSize()) - if !usable { - reasons := unusableResult.reasons() + // check if the host is still usable + if !check.Usability.IsUsable() { + reasons := check.Usability.UnusableReasons() toStopUsing[fcid] = strings.Join(reasons, ",") c.logger.Infow("unusable host", "hk", hk, "fcid", fcid, "reasons", reasons) continue @@ -805,7 +760,8 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts if _, found := inCurrentSet[fcid]; !found || remainingKeepLeeway == 0 { toStopUsing[fcid] = errContractNoRevision.Error() } else if !state.cfg.Hosts.AllowRedundantIPs && ipFilter.IsRedundantIP(contract.HostIP, contract.HostKey) { - toStopUsing[fcid] = fmt.Sprintf("%v; %v", errHostRedundantIP, errContractNoRevision) + toStopUsing[fcid] = fmt.Sprintf("%v; %v", api.ErrUsabilityHostRedundantIP, errContractNoRevision) + hostChecks[contract.HostKey].Usability.RedundantIP = true } else { toKeep = append(toKeep, contract.ContractMetadata) remainingKeepLeeway-- // we let it slide @@ -813,21 +769,9 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts continue // can't perform contract checks without revision } - // if we were not able to get a valid price table for the host, but we - // did pass the host checks, we only want to be lenient if this contract - // is in the current set and only for a certain number of times, - // controlled by maxKeepLeeway - if invalidPT { - if _, found := inCurrentSet[fcid]; !found || remainingKeepLeeway == 0 { - toStopUsing[fcid] = "no valid price table" - continue - } - remainingKeepLeeway-- // we let it slide - } - // decide whether the contract is still good ci := contractInfo{contract: contract, priceTable: host.PriceTable.HostPriceTable, settings: host.Settings} - usable, recoverable, refresh, renew, reasons := c.isUsableContract(state.cfg, state, ci, cs.BlockHeight, ipFilter) + usable, recoverable, refresh, renew, reasons := c.isUsableContract(state.cfg, state, ci, bh, ipFilter) ci.usable = usable ci.recoverable = recoverable if !usable { @@ -854,10 +798,32 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts } } - return toKeep, toArchive, toStopUsing, toRefresh, toRenew, nil + return toKeep, toArchive, toStopUsing, toRefresh, toRenew } -func (c *contractor) runContractFormations(ctx context.Context, w Worker, candidates scoredHosts, usedHosts map[types.PublicKey]struct{}, unusableHosts unusableHostResult, missing uint64, budget *types.Currency) (formed []api.ContractMetadata, _ error) { +func (c *contractor) runHostChecks(ctx context.Context, hosts []api.Host, hostData map[types.PublicKey]uint64, minScore float64) (map[types.PublicKey]*api.HostCheck, error) { + // convenience variables + state := c.ap.State() + + // fetch consensus state + cs, err := c.ap.bus.ConsensusState(ctx) + if err != nil { + return nil, err + } + + // create gouging checker + gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) + + // check all hosts + checks := make(map[types.PublicKey]*api.HostCheck) + for _, h := range hosts { + h.PriceTable.HostBlockHeight = cs.BlockHeight // ignore HostBlockHeight + checks[h.PublicKey] = checkHost(state.cfg, state.rs, gc, h, minScore, hostData[h.PublicKey]) + } + return checks, nil +} + +func (c *contractor) runContractFormations(ctx context.Context, w Worker, candidates scoredHosts, usedHosts map[types.PublicKey]struct{}, unusableHosts unusableHostsBreakdown, missing uint64, budget *types.Currency) (formed []api.ContractMetadata, _ error) { if c.ap.isStopped() { return nil, nil } @@ -1310,13 +1276,13 @@ func (c *contractor) calculateMinScore(candidates []scoredHost, numContracts uin return minScore } -func (c *contractor) candidateHosts(ctx context.Context, hosts []api.Host, usedHosts map[types.PublicKey]struct{}, storedData map[types.PublicKey]uint64, minScore float64) ([]scoredHost, unusableHostResult, error) { +func (c *contractor) candidateHosts(ctx context.Context, hosts []api.Host, usedHosts map[types.PublicKey]struct{}, storedData map[types.PublicKey]uint64, minScore float64) ([]scoredHost, unusableHostsBreakdown, error) { start := time.Now() // fetch consensus state cs, err := c.ap.bus.ConsensusState(ctx) if err != nil { - return nil, unusableHostResult{}, err + return nil, unusableHostsBreakdown{}, err } // create a gouging checker @@ -1325,13 +1291,18 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []api.Host, usedH // select unused hosts that passed a scan var unused []api.Host - var excluded, notcompletedscan int + var blocked, excluded, notcompletedscan int for _, h := range hosts { // filter out used hosts if _, exclude := usedHosts[h.PublicKey]; exclude { excluded++ continue } + // filter out blocked hosts + if h.Blocked { + blocked++ + continue + } // filter out unscanned hosts if !h.Scanned { notcompletedscan++ @@ -1346,7 +1317,7 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []api.Host, usedH "used", len(usedHosts)) // score all unused hosts - var unusableHostResult unusableHostResult + var unusableHosts unusableHostsBreakdown var unusable, zeros int var candidates []scoredHost for _, h := range unused { @@ -1359,15 +1330,15 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []api.Host, usedH // NOTE: ignore the pricetable's HostBlockHeight by setting it to our // own blockheight h.PriceTable.HostBlockHeight = cs.BlockHeight - usable, result := isUsableHost(state.cfg, state.rs, gc, h, minScore, storedData[h.PublicKey]) - if usable { - candidates = append(candidates, scoredHost{h.Host, result.scoreBreakdown.Score()}) + hc := checkHost(state.cfg, state.rs, gc, h, minScore, storedData[h.PublicKey]) + if hc.Usability.IsUsable() { + candidates = append(candidates, scoredHost{h, hc.Score.Score()}) continue } // keep track of unusable host results - unusableHostResult.merge(result) - if result.scoreBreakdown.Score() == 0 { + unusableHosts.track(hc.Usability) + if hc.Score.Score() == 0 { zeros++ } unusable++ @@ -1378,7 +1349,7 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []api.Host, usedH "unusable", unusable, "used", len(usedHosts)) - return candidates, unusableHostResult, nil + return candidates, unusableHosts, nil } func (c *contractor) renewContract(ctx context.Context, w Worker, ci contractInfo, budget *types.Currency) (cm api.ContractMetadata, proceed bool, err error) { @@ -1543,7 +1514,7 @@ func (c *contractor) refreshContract(ctx context.Context, w Worker, ci contractI return refreshedContract, true, nil } -func (c *contractor) formContract(ctx context.Context, w Worker, host hostdb.Host, minInitialContractFunds, maxInitialContractFunds types.Currency, budget *types.Currency) (cm api.ContractMetadata, proceed bool, err error) { +func (c *contractor) formContract(ctx context.Context, w Worker, host api.Host, minInitialContractFunds, maxInitialContractFunds types.Currency, budget *types.Currency) (cm api.ContractMetadata, proceed bool, err error) { // convenience variables state := c.ap.State() hk := host.PublicKey @@ -1691,7 +1662,7 @@ func initialContractFundingMinMax(cfg api.AutopilotConfig) (min types.Currency, return } -func refreshPriceTable(ctx context.Context, w Worker, host *hostdb.Host) error { +func refreshPriceTable(ctx context.Context, w Worker, host *api.Host) error { // return early if the host's pricetable is not expired yet if time.Now().Before(host.PriceTable.Expiry) { return nil diff --git a/autopilot/host_test.go b/autopilot/host_test.go index fa1a0ab44..965b2b05a 100644 --- a/autopilot/host_test.go +++ b/autopilot/host_test.go @@ -8,7 +8,7 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" - "go.sia.tech/renterd/hostdb" + "go.sia.tech/renterd/api" "lukechampine.com/frand" ) @@ -40,20 +40,20 @@ func TestHost(t *testing.T) { } } -func newTestHosts(n int) []hostdb.Host { - hosts := make([]hostdb.Host, n) +func newTestHosts(n int) []api.Host { + hosts := make([]api.Host, n) for i := 0; i < n; i++ { hosts[i] = newTestHost(randomHostKey(), newTestHostPriceTable(), newTestHostSettings()) } return hosts } -func newTestHost(hk types.PublicKey, pt rhpv3.HostPriceTable, settings rhpv2.HostSettings) hostdb.Host { - return hostdb.Host{ +func newTestHost(hk types.PublicKey, pt rhpv3.HostPriceTable, settings rhpv2.HostSettings) api.Host { + return api.Host{ NetAddress: randomIP().String(), KnownSince: time.Now(), LastAnnouncement: time.Now(), - Interactions: hostdb.Interactions{ + Interactions: api.HostInteractions{ TotalScans: 2, LastScan: time.Now().Add(-time.Minute), LastScanSuccess: true, @@ -65,7 +65,7 @@ func newTestHost(hk types.PublicKey, pt rhpv3.HostPriceTable, settings rhpv2.Hos FailedInteractions: 0, }, PublicKey: hk, - PriceTable: hostdb.HostPriceTable{HostPriceTable: pt, Expiry: time.Now().Add(time.Minute)}, + PriceTable: api.HostPriceTable{HostPriceTable: pt, Expiry: time.Now().Add(time.Minute)}, Settings: settings, Scanned: true, } diff --git a/autopilot/hostfilter.go b/autopilot/hostfilter.go index d64c1f3e3..15a2147ab 100644 --- a/autopilot/hostfilter.go +++ b/autopilot/hostfilter.go @@ -5,7 +5,6 @@ import ( "fmt" "math" "math/big" - "strings" rhpv2 "go.sia.tech/core/rhp/v2" rhpv3 "go.sia.tech/core/rhp/v3" @@ -31,16 +30,6 @@ const ( ) var ( - errHostBlocked = errors.New("host is blocked") - errHostNotFound = errors.New("host not found") - errHostOffline = errors.New("host is offline") - errLowScore = errors.New("host's score is below minimum") - errHostRedundantIP = errors.New("host has redundant IP") - errHostPriceGouging = errors.New("host is price gouging") - errHostNotAcceptingContracts = errors.New("host is not accepting contracts") - errHostNotCompletingScan = errors.New("host is not completing scan") - errHostNotAnnounced = errors.New("host is not announced") - errContractOutOfCollateral = errors.New("contract is out of collateral") errContractOutOfFunds = errors.New("contract is out of funds") errContractUpForRenewal = errors.New("contract is up for renewal") @@ -50,7 +39,7 @@ var ( errContractNotConfirmed = errors.New("contract hasn't been confirmed on chain in time") ) -type unusableHostResult struct { +type unusableHostsBreakdown struct { blocked uint64 offline uint64 lowscore uint64 @@ -59,100 +48,36 @@ type unusableHostResult struct { notacceptingcontracts uint64 notannounced uint64 notcompletingscan uint64 - unknown uint64 - - // gougingBreakdown is mostly ignored, we overload the unusableHostResult - // with a gouging breakdown to be able to return it in the host infos - // endpoint `/hosts/:hostkey` - gougingBreakdown api.HostGougingBreakdown - - // scoreBreakdown is mostly ignored, we overload the unusableHostResult with - // a score breakdown to be able to return it in the host infos endpoint - // `/hosts/:hostkey` - scoreBreakdown api.HostScoreBreakdown -} - -func newUnusableHostResult(errs []error, gougingBreakdown api.HostGougingBreakdown, scoreBreakdown api.HostScoreBreakdown) (u unusableHostResult) { - for _, err := range errs { - if errors.Is(err, errHostBlocked) { - u.blocked++ - } else if errors.Is(err, errHostOffline) { - u.offline++ - } else if errors.Is(err, errLowScore) { - u.lowscore++ - } else if errors.Is(err, errHostRedundantIP) { - u.redundantip++ - } else if errors.Is(err, errHostPriceGouging) { - u.gouging++ - } else if errors.Is(err, errHostNotAcceptingContracts) { - u.notacceptingcontracts++ - } else if errors.Is(err, errHostNotAnnounced) { - u.notannounced++ - } else if errors.Is(err, errHostNotCompletingScan) { - u.notcompletingscan++ - } else { - u.unknown++ - } - } - - u.gougingBreakdown = gougingBreakdown - u.scoreBreakdown = scoreBreakdown - return -} - -func (u unusableHostResult) String() string { - return fmt.Sprintf("host is unusable because of the following reasons: %v", strings.Join(u.reasons(), ", ")) } -func (u unusableHostResult) reasons() []string { - var reasons []string - if u.blocked > 0 { - reasons = append(reasons, errHostBlocked.Error()) - } - if u.offline > 0 { - reasons = append(reasons, errHostOffline.Error()) +func (u *unusableHostsBreakdown) track(ub api.HostUsabilityBreakdown) { + if ub.Blocked { + u.blocked++ } - if u.lowscore > 0 { - reasons = append(reasons, errLowScore.Error()) + if ub.Offline { + u.offline++ } - if u.redundantip > 0 { - reasons = append(reasons, errHostRedundantIP.Error()) + if ub.LowScore { + u.lowscore++ } - if u.gouging > 0 { - reasons = append(reasons, errHostPriceGouging.Error()) + if ub.RedundantIP { + u.redundantip++ } - if u.notacceptingcontracts > 0 { - reasons = append(reasons, errHostNotAcceptingContracts.Error()) + if ub.Gouging { + u.gouging++ } - if u.notannounced > 0 { - reasons = append(reasons, errHostNotAnnounced.Error()) + if ub.NotAcceptingContracts { + u.notacceptingcontracts++ } - if u.notcompletingscan > 0 { - reasons = append(reasons, errHostNotCompletingScan.Error()) + if ub.NotAnnounced { + u.notannounced++ } - if u.unknown > 0 { - reasons = append(reasons, "unknown") + if ub.NotCompletingScan { + u.notcompletingscan++ } - return reasons } -func (u *unusableHostResult) merge(other unusableHostResult) { - u.blocked += other.blocked - u.offline += other.offline - u.lowscore += other.lowscore - u.redundantip += other.redundantip - u.gouging += other.gouging - u.notacceptingcontracts += other.notacceptingcontracts - u.notannounced += other.notannounced - u.notcompletingscan += other.notcompletingscan - u.unknown += other.unknown - - // scoreBreakdown is not merged - // - // gougingBreakdown is not merged -} - -func (u *unusableHostResult) keysAndValues() []interface{} { +func (u *unusableHostsBreakdown) keysAndValues() []interface{} { values := []interface{}{ "blocked", u.blocked, "offline", u.offline, @@ -162,7 +87,6 @@ func (u *unusableHostResult) keysAndValues() []interface{} { "notacceptingcontracts", u.notacceptingcontracts, "notcompletingscan", u.notcompletingscan, "notannounced", u.notannounced, - "unknown", u.unknown, } for i := 0; i < len(values); i += 2 { if values[i+1].(uint64) == 0 { @@ -173,39 +97,42 @@ func (u *unusableHostResult) keysAndValues() []interface{} { return values } -// isUsableHost returns whether the given host is usable along with a list of -// reasons why it was deemed unusable. -func isUsableHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.GougingChecker, h api.Host, minScore float64, storedData uint64) (bool, unusableHostResult) { +// checkHost performs a series of checks on the host. +func checkHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.GougingChecker, h api.Host, minScore float64, storedData uint64) *api.HostCheck { if rs.Validate() != nil { panic("invalid redundancy settings were supplied - developer error") } - var errs []error + // prepare host breakdown fields + var gb api.HostGougingBreakdown + var sb api.HostScoreBreakdown + var ub api.HostUsabilityBreakdown + + // blocked status does not influence what host info is calculated if h.Blocked { - errs = append(errs, errHostBlocked) + ub.Blocked = true } - var gougingBreakdown api.HostGougingBreakdown - var scoreBreakdown api.HostScoreBreakdown + // calculate remaining host info fields if !h.IsAnnounced() { - errs = append(errs, errHostNotAnnounced) + ub.NotAnnounced = true } else if !h.Scanned { - errs = append(errs, errHostNotCompletingScan) + ub.NotCompletingScan = true } else { // online check if !h.IsOnline() { - errs = append(errs, errHostOffline) + ub.Offline = true } // accepting contracts check if !h.Settings.AcceptingContracts { - errs = append(errs, errHostNotAcceptingContracts) + ub.NotAcceptingContracts = true } // perform gouging checks - gougingBreakdown = gc.Check(&h.Settings, &h.PriceTable.HostPriceTable) - if gougingBreakdown.Gouging() { - errs = append(errs, fmt.Errorf("%w: %v", errHostPriceGouging, gougingBreakdown)) + gb = gc.Check(&h.Settings, &h.PriceTable.HostPriceTable) + if gb.Gouging() { + ub.Gouging = true } else if minScore > 0 { // perform scoring checks // @@ -213,14 +140,18 @@ func isUsableHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker. // not gouging, this because the core package does not have overflow // checks in its cost calculations needed to calculate the period // cost - scoreBreakdown = hostScore(cfg, h.Host, storedData, rs.Redundancy()) - if scoreBreakdown.Score() < minScore { - errs = append(errs, fmt.Errorf("%w: (%s): %v < %v", errLowScore, scoreBreakdown.String(), scoreBreakdown.Score(), minScore)) + sb = hostScore(cfg, h, storedData, rs.Redundancy()) + if sb.Score() < minScore { + ub.LowScore = true } } } - return len(errs) == 0, newUnusableHostResult(errs, gougingBreakdown, scoreBreakdown) + return &api.HostCheck{ + Usability: ub, + Gouging: gb, + Score: sb, + } } // isUsableContract returns whether the given contract is @@ -271,7 +202,7 @@ func (c *contractor) isUsableContract(cfg api.AutopilotConfig, state state, ci c // IP check should be last since it modifies the filter shouldFilter := !cfg.Hosts.AllowRedundantIPs && (usable || recoverable) if shouldFilter && f.IsRedundantIP(contract.HostIP, contract.HostKey) { - reasons = append(reasons, errHostRedundantIP.Error()) + reasons = append(reasons, api.ErrUsabilityHostRedundantIP.Error()) usable = false recoverable = false // do not use in the contract set, but keep it around for downloads renew = false // do not renew, but allow refreshes so the contracts stays funded diff --git a/autopilot/hostinfo.go b/autopilot/hostinfo.go deleted file mode 100644 index d82062a80..000000000 --- a/autopilot/hostinfo.go +++ /dev/null @@ -1,205 +0,0 @@ -package autopilot - -import ( - "context" - "fmt" - - "go.sia.tech/core/types" - "go.sia.tech/renterd/api" - "go.sia.tech/renterd/worker" -) - -func (c *contractor) HostInfo(ctx context.Context, hostKey types.PublicKey) (api.HostHandlerResponse, error) { - state := c.ap.State() - - if state.cfg.Contracts.Allowance.IsZero() { - return api.HostHandlerResponse{}, fmt.Errorf("can not score hosts because contracts allowance is zero") - } - if state.cfg.Contracts.Amount == 0 { - return api.HostHandlerResponse{}, fmt.Errorf("can not score hosts because contracts amount is zero") - } - if state.cfg.Contracts.Period == 0 { - return api.HostHandlerResponse{}, fmt.Errorf("can not score hosts because contract period is zero") - } - - host, err := c.ap.bus.Host(ctx, hostKey) - if err != nil { - return api.HostHandlerResponse{}, fmt.Errorf("failed to fetch requested host from bus: %w", err) - } - gs, err := c.ap.bus.GougingSettings(ctx) - if err != nil { - return api.HostHandlerResponse{}, fmt.Errorf("failed to fetch gouging settings from bus: %w", err) - } - rs, err := c.ap.bus.RedundancySettings(ctx) - if err != nil { - return api.HostHandlerResponse{}, fmt.Errorf("failed to fetch redundancy settings from bus: %w", err) - } - cs, err := c.ap.bus.ConsensusState(ctx) - if err != nil { - return api.HostHandlerResponse{}, fmt.Errorf("failed to fetch consensus state from bus: %w", err) - } - fee, err := c.ap.bus.RecommendedFee(ctx) - if err != nil { - return api.HostHandlerResponse{}, fmt.Errorf("failed to fetch recommended fee from bus: %w", err) - } - c.mu.Lock() - storedData := c.cachedDataStored[hostKey] - minScore := c.cachedMinScore - c.mu.Unlock() - - gc := worker.NewGougingChecker(gs, cs, fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) - - // ignore the pricetable's HostBlockHeight by setting it to our own blockheight - host.Host.PriceTable.HostBlockHeight = cs.BlockHeight - - isUsable, unusableResult := isUsableHost(state.cfg, rs, gc, host, minScore, storedData) - return api.HostHandlerResponse{ - Host: host.Host, - Checks: &api.HostHandlerResponseChecks{ - Gouging: unusableResult.gougingBreakdown.Gouging(), - GougingBreakdown: unusableResult.gougingBreakdown, - Score: unusableResult.scoreBreakdown.Score(), - ScoreBreakdown: unusableResult.scoreBreakdown, - Usable: isUsable, - UnusableReasons: unusableResult.reasons(), - }, - }, nil -} - -func (c *contractor) hostInfoFromCache(ctx context.Context, host api.Host) (hi hostInfo, found bool) { - // grab host details from cache - c.mu.Lock() - hi, found = c.cachedHostInfo[host.PublicKey] - storedData := c.cachedDataStored[host.PublicKey] - minScore := c.cachedMinScore - c.mu.Unlock() - - // return early if the host info is not cached - if !found { - return - } - - // try and refresh the host info if it got scanned in the meantime, this - // inconsistency would resolve itself but trying to update it here improves - // first time user experience - if host.Scanned && hi.UnusableResult.notcompletingscan > 0 { - cs, err := c.ap.bus.ConsensusState(ctx) - if err != nil { - c.logger.Error("failed to fetch consensus state from bus: %v", err) - } else { - state := c.ap.State() - gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) - isUsable, unusableResult := isUsableHost(state.cfg, state.rs, gc, host, minScore, storedData) - hi = hostInfo{ - Usable: isUsable, - UnusableResult: unusableResult, - } - - // update cache - c.mu.Lock() - c.cachedHostInfo[host.PublicKey] = hi - c.mu.Unlock() - } - } - - return -} - -func (c *contractor) HostInfos(ctx context.Context, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.HostHandlerResponse, error) { - // declare helper to decide whether to keep a host. - if !isValidUsabilityFilterMode(usabilityMode) { - return nil, fmt.Errorf("invalid usability mode: '%v', options are 'usable', 'unusable' or an empty string for no filter", usabilityMode) - } - - keep := func(usable bool) bool { - switch usabilityMode { - case api.UsabilityFilterModeUsable: - return usable // keep usable - case api.UsabilityFilterModeUnusable: - return !usable // keep unusable - case api.UsabilityFilterModeAll: - return true // keep all - case "": - return true // keep all - default: - panic("unreachable") - } - } - - var hostInfos []api.HostHandlerResponse - wanted := limit - for { - // fetch up to 'limit' hosts. - hosts, err := c.ap.bus.SearchHosts(ctx, api.SearchHostOptions{ - Offset: offset, - Limit: limit, - FilterMode: filterMode, - AddressContains: addressContains, - KeyIn: keyIn, - }) - if err != nil { - return nil, err - } - offset += len(hosts) - - // if there are no more hosts, we're done. - if len(hosts) == 0 { - return hostInfos, nil // no more hosts - } - - // decide how many of the returned hosts to keep. - var keptHosts int - for _, host := range hosts { - hi, cached := c.hostInfoFromCache(ctx, host) - if !cached { - // when the filterMode is "all" we include uncached hosts and - // set IsChecked = false. - if usabilityMode == api.UsabilityFilterModeAll { - hostInfos = append(hostInfos, api.HostHandlerResponse{ - Host: host.Host, - }) - if wanted > 0 && len(hostInfos) == wanted { - return hostInfos, nil // we're done. - } - keptHosts++ - } - continue - } - if !keep(hi.Usable) { - continue - } - hostInfos = append(hostInfos, api.HostHandlerResponse{ - Host: host.Host, - Checks: &api.HostHandlerResponseChecks{ - Gouging: hi.UnusableResult.gougingBreakdown.Gouging(), - GougingBreakdown: hi.UnusableResult.gougingBreakdown, - Score: hi.UnusableResult.scoreBreakdown.Score(), - ScoreBreakdown: hi.UnusableResult.scoreBreakdown, - Usable: hi.Usable, - UnusableReasons: hi.UnusableResult.reasons(), - }, - }) - if wanted > 0 && len(hostInfos) == wanted { - return hostInfos, nil // we're done. - } - keptHosts++ - } - - // if no hosts were kept from this batch, double the limit. - if limit > 0 && keptHosts == 0 { - limit *= 2 - } - } -} - -func isValidUsabilityFilterMode(usabilityMode string) bool { - switch usabilityMode { - case api.UsabilityFilterModeUsable: - case api.UsabilityFilterModeUnusable: - case api.UsabilityFilterModeAll: - case "": - default: - return false - } - return true -} diff --git a/autopilot/hosts_test.go b/autopilot/hosts_test.go index 332bf1ea3..6644a1cd2 100644 --- a/autopilot/hosts_test.go +++ b/autopilot/hosts_test.go @@ -5,7 +5,7 @@ import ( "testing" "go.sia.tech/core/types" - "go.sia.tech/renterd/hostdb" + "go.sia.tech/renterd/api" "lukechampine.com/frand" ) @@ -18,7 +18,7 @@ func TestScoredHostsRandSelectByScore(t *testing.T) { var hosts scoredHosts for hk, score := range hostToScores { - hosts = append(hosts, scoredHost{score: score, host: hostdb.Host{PublicKey: hk}}) + hosts = append(hosts, scoredHost{score: score, host: api.Host{PublicKey: hk}}) } for i := 0; i < 1000; i++ { @@ -55,8 +55,8 @@ func TestScoredHostsRandSelectByScore(t *testing.T) { // assert select is random on equal inputs counts := make([]int, 2) hosts = scoredHosts{ - {score: .1, host: hostdb.Host{PublicKey: types.PublicKey{1}}}, - {score: .1, host: hostdb.Host{PublicKey: types.PublicKey{2}}}, + {score: .1, host: api.Host{PublicKey: types.PublicKey{1}}}, + {score: .1, host: api.Host{PublicKey: types.PublicKey{2}}}, } for i := 0; i < 100; i++ { if hosts.randSelectByScore(1)[0].host.PublicKey == (types.PublicKey{1}) { diff --git a/autopilot/hostscore.go b/autopilot/hostscore.go index 3c26dce42..fc98499f1 100644 --- a/autopilot/hostscore.go +++ b/autopilot/hostscore.go @@ -9,13 +9,12 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/siad/build" ) const smallestValidScore = math.SmallestNonzeroFloat64 -func hostScore(cfg api.AutopilotConfig, h hostdb.Host, storedData uint64, expectedRedundancy float64) api.HostScoreBreakdown { +func hostScore(cfg api.AutopilotConfig, h api.Host, storedData uint64, expectedRedundancy float64) api.HostScoreBreakdown { // idealDataPerHost is the amount of data that we would have to put on each // host assuming that our storage requirements were spread evenly across // every single host. @@ -92,7 +91,7 @@ func storageRemainingScore(h rhpv2.HostSettings, storedData uint64, allocationPe return math.Pow(storageRatio, 2.0) } -func ageScore(h hostdb.Host) float64 { +func ageScore(h api.Host) float64 { // sanity check if h.KnownSince.IsZero() { return 0 @@ -179,14 +178,14 @@ func collateralScore(cfg api.AutopilotConfig, pt rhpv3.HostPriceTable, allocatio } } -func interactionScore(h hostdb.Host) float64 { +func interactionScore(h api.Host) float64 { success, fail := 30.0, 1.0 success += h.Interactions.SuccessfulInteractions fail += h.Interactions.FailedInteractions return math.Pow(success/(success+fail), 10) } -func uptimeScore(h hostdb.Host) float64 { +func uptimeScore(h api.Host) float64 { secondToLastScanSuccess := h.Interactions.SecondToLastScanSuccess lastScanSuccess := h.Interactions.LastScanSuccess uptime := h.Interactions.Uptime @@ -258,7 +257,7 @@ func versionScore(settings rhpv2.HostSettings) float64 { // contractPriceForScore returns the contract price of the host used for // scoring. Since we don't know whether rhpv2 or rhpv3 are used, we return the // bigger one for a pesimistic score. -func contractPriceForScore(h hostdb.Host) types.Currency { +func contractPriceForScore(h api.Host) types.Currency { cp := h.Settings.ContractPrice if cp.Cmp(h.PriceTable.ContractPrice) > 0 { cp = h.PriceTable.ContractPrice @@ -285,26 +284,26 @@ func sectorUploadCost(pt rhpv3.HostPriceTable, duration uint64) types.Currency { return uploadSectorCostRHPv3 } -func uploadCostForScore(cfg api.AutopilotConfig, h hostdb.Host, bytes uint64) types.Currency { +func uploadCostForScore(cfg api.AutopilotConfig, h api.Host, bytes uint64) types.Currency { uploadSectorCostRHPv3 := sectorUploadCost(h.PriceTable.HostPriceTable, cfg.Contracts.Period) numSectors := bytesToSectors(bytes) return uploadSectorCostRHPv3.Mul64(numSectors) } -func downloadCostForScore(h hostdb.Host, bytes uint64) types.Currency { +func downloadCostForScore(h api.Host, bytes uint64) types.Currency { rsc := h.PriceTable.BaseCost().Add(h.PriceTable.ReadSectorCost(rhpv2.SectorSize)) downloadSectorCostRHPv3, _ := rsc.Total() numSectors := bytesToSectors(bytes) return downloadSectorCostRHPv3.Mul64(numSectors) } -func storageCostForScore(cfg api.AutopilotConfig, h hostdb.Host, bytes uint64) types.Currency { +func storageCostForScore(cfg api.AutopilotConfig, h api.Host, bytes uint64) types.Currency { storeSectorCostRHPv3 := sectorStorageCost(h.PriceTable.HostPriceTable, cfg.Contracts.Period) numSectors := bytesToSectors(bytes) return storeSectorCostRHPv3.Mul64(numSectors) } -func hostPeriodCostForScore(h hostdb.Host, cfg api.AutopilotConfig, expectedRedundancy float64) types.Currency { +func hostPeriodCostForScore(h api.Host, cfg api.AutopilotConfig, expectedRedundancy float64) types.Currency { // compute how much data we upload, download and store. uploadPerHost := uint64(float64(cfg.Contracts.Upload) * expectedRedundancy / float64(cfg.Contracts.Amount)) downloadPerHost := uint64(float64(cfg.Contracts.Download) * expectedRedundancy / float64(cfg.Contracts.Amount)) diff --git a/autopilot/hostscore_test.go b/autopilot/hostscore_test.go index e48417235..bc5f61b8a 100644 --- a/autopilot/hostscore_test.go +++ b/autopilot/hostscore_test.go @@ -9,7 +9,6 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" ) var cfg = api.AutopilotConfig{ @@ -34,7 +33,7 @@ var cfg = api.AutopilotConfig{ func TestHostScore(t *testing.T) { day := 24 * time.Hour - newHost := func(s rhpv2.HostSettings) hostdb.Host { + newHost := func(s rhpv2.HostSettings) api.Host { return newTestHost(randomHostKey(), newTestHostPriceTable(), s) } h1 := newHost(newTestHostSettings()) diff --git a/autopilot/scanner.go b/autopilot/scanner.go index d733c8d0c..f466a842d 100644 --- a/autopilot/scanner.go +++ b/autopilot/scanner.go @@ -11,7 +11,6 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/internal/utils" "go.uber.org/zap" ) @@ -32,7 +31,7 @@ type ( // scanner tests with every interface change bus interface { SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.Host, error) - HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error) + HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]api.HostAddress, error) RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) } diff --git a/autopilot/scanner_test.go b/autopilot/scanner_test.go index 860a855fe..512cbd517 100644 --- a/autopilot/scanner_test.go +++ b/autopilot/scanner_test.go @@ -9,13 +9,12 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) type mockBus struct { - hosts []hostdb.Host + hosts []api.Host reqs []string } @@ -32,14 +31,10 @@ func (b *mockBus) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ( end = len(b.hosts) } - hosts := make([]api.Host, len(b.hosts[start:end])) - for i, h := range b.hosts[start:end] { - hosts[i] = api.Host{Host: h} - } - return hosts, nil + return b.hosts[start:end], nil } -func (b *mockBus) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error) { +func (b *mockBus) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]api.HostAddress, error) { hosts, err := b.SearchHosts(ctx, api.SearchHostOptions{ Offset: opts.Offset, Limit: opts.Limit, @@ -47,9 +42,9 @@ func (b *mockBus) HostsForScanning(ctx context.Context, opts api.HostsForScannin if err != nil { return nil, err } - var hostAddresses []hostdb.HostAddress + var hostAddresses []api.HostAddress for _, h := range hosts { - hostAddresses = append(hostAddresses, hostdb.HostAddress{ + hostAddresses = append(hostAddresses, api.HostAddress{ NetAddress: h.NetAddress, PublicKey: h.PublicKey, }) @@ -80,8 +75,8 @@ func (w *mockWorker) RHPScan(ctx context.Context, hostKey types.PublicKey, hostI return api.RHPScanResponse{}, nil } -func (w *mockWorker) RHPPriceTable(ctx context.Context, hostKey types.PublicKey, siamuxAddr string) (hostdb.HostPriceTable, error) { - return hostdb.HostPriceTable{}, nil +func (w *mockWorker) RHPPriceTable(ctx context.Context, hostKey types.PublicKey, siamuxAddr string) (api.HostPriceTable, error) { + return api.HostPriceTable{}, nil } func TestScanner(t *testing.T) { diff --git a/autopilot/workerpool.go b/autopilot/workerpool.go index d8c821354..16a6b4c99 100644 --- a/autopilot/workerpool.go +++ b/autopilot/workerpool.go @@ -9,7 +9,6 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/object" "lukechampine.com/frand" ) @@ -23,7 +22,7 @@ type Worker interface { RHPBroadcast(ctx context.Context, fcid types.FileContractID) (err error) RHPForm(ctx context.Context, endHeight uint64, hk types.PublicKey, hostIP string, renterAddress types.Address, renterFunds types.Currency, hostCollateral types.Currency) (rhpv2.ContractRevision, []types.Transaction, error) RHPFund(ctx context.Context, contractID types.FileContractID, hostKey types.PublicKey, hostIP, siamuxAddr string, balance types.Currency) (err error) - RHPPriceTable(ctx context.Context, hostKey types.PublicKey, siamuxAddr string, timeout time.Duration) (hostdb.HostPriceTable, error) + RHPPriceTable(ctx context.Context, hostKey types.PublicKey, siamuxAddr string, timeout time.Duration) (api.HostPriceTable, error) RHPPruneContract(ctx context.Context, fcid types.FileContractID, timeout time.Duration) (pruned, remaining uint64, err error) RHPRenew(ctx context.Context, fcid types.FileContractID, endHeight uint64, hk types.PublicKey, hostIP string, hostAddress, renterAddress types.Address, renterFunds, minNewCollateral types.Currency, expectedStorage, windowSize uint64) (api.RHPRenewResponse, error) RHPScan(ctx context.Context, hostKey types.PublicKey, hostIP string, timeout time.Duration) (api.RHPScanResponse, error) diff --git a/bus/bus.go b/bus/bus.go index 8c7c99649..a6b86c0e1 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -23,7 +23,6 @@ import ( "go.sia.tech/renterd/api" "go.sia.tech/renterd/build" "go.sia.tech/renterd/bus/client" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/object" "go.sia.tech/renterd/wallet" "go.sia.tech/renterd/webhooks" @@ -92,17 +91,17 @@ type ( // A HostDB stores information about hosts. HostDB interface { Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) - HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]hostdb.HostAddress, error) - RecordHostScans(ctx context.Context, scans []hostdb.HostScan) error - RecordPriceTables(ctx context.Context, priceTableUpdate []hostdb.PriceTableUpdate) error - RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) - ResetLostSectors(ctx context.Context, hk types.PublicKey) error - SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.Host, error) - HostAllowlist(ctx context.Context) ([]types.PublicKey, error) HostBlocklist(ctx context.Context) ([]string, error) + HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]api.HostAddress, error) + RecordHostScans(ctx context.Context, scans []api.HostScan) error + RecordPriceTables(ctx context.Context, priceTableUpdate []api.HostPriceTableUpdate) error + RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) + ResetLostSectors(ctx context.Context, hk types.PublicKey) error + SearchHosts(ctx context.Context, autopilotID, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.Host, error) UpdateHostAllowlistEntries(ctx context.Context, add, remove []types.PublicKey, clear bool) error UpdateHostBlocklistEntries(ctx context.Context, add, remove []string, clear bool) error + UpdateHostCheck(ctx context.Context, autopilotID string, hk types.PublicKey, check api.HostCheck) error } // A MetadataStore stores information about contracts and objects. @@ -253,6 +252,8 @@ func (b *bus) Handler() http.Handler { "GET /autopilot/:id": b.autopilotsHandlerGET, "PUT /autopilot/:id": b.autopilotsHandlerPUT, + "PUT /autopilot/:id/host/:hostkey/check": b.autopilotHostCheckHandlerPUT, + "GET /buckets": b.bucketsHandlerGET, "POST /buckets": b.bucketsHandlerPOST, "PUT /bucket/:name/policy": b.bucketsHandlerPolicyPUT, @@ -762,7 +763,7 @@ func (b *bus) hostsHandlerGETDeprecated(jc jape.Context) { } // fetch hosts - hosts, err := b.hdb.SearchHosts(jc.Request.Context(), api.HostFilterModeAllowed, "", nil, offset, limit) + hosts, err := b.hdb.SearchHosts(jc.Request.Context(), "", api.HostFilterModeAllowed, api.UsabilityFilterModeAll, "", nil, offset, limit) if jc.Check(fmt.Sprintf("couldn't fetch hosts %d-%d", offset, offset+limit), err) != nil { return } @@ -776,9 +777,10 @@ func (b *bus) searchHostsHandlerPOST(jc jape.Context) { } // TODO: on the next major release: - // - properly default search params - // - properly validate and return 400 - hosts, err := b.hdb.SearchHosts(jc.Request.Context(), req.FilterMode, req.AddressContains, req.KeyIn, req.Offset, req.Limit) + // - properly default search params (currently no defaults are set) + // - properly validate and return 400 (currently validation is done in autopilot and the store) + + hosts, err := b.hdb.SearchHosts(jc.Request.Context(), req.AutopilotID, req.FilterMode, req.UsabilityMode, req.AddressContains, req.KeyIn, req.Offset, req.Limit) if jc.Check(fmt.Sprintf("couldn't fetch hosts %d-%d", req.Offset, req.Offset+req.Limit), err) != nil { return } @@ -1967,6 +1969,29 @@ func (b *bus) autopilotsHandlerPUT(jc jape.Context) { jc.Check("failed to update autopilot", b.as.UpdateAutopilot(jc.Request.Context(), ap)) } +func (b *bus) autopilotHostCheckHandlerPUT(jc jape.Context) { + var id string + if jc.DecodeParam("id", &id) != nil { + return + } + var hk types.PublicKey + if jc.DecodeParam("hostkey", &hk) != nil { + return + } + var hc api.HostCheck + if jc.Check("failed to decode host check", jc.Decode(&hc)) != nil { + return + } + + err := b.hdb.UpdateHostCheck(jc.Request.Context(), id, hk, hc) + if errors.Is(err, api.ErrAutopilotNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("failed to update host", err) != nil { + return + } +} + func (b *bus) contractTaxHandlerGET(jc jape.Context) { var payout types.Currency if jc.DecodeParam("payout", (*api.ParamCurrency)(&payout)) != nil { diff --git a/bus/client/hosts.go b/bus/client/hosts.go index 8338d53f7..709cb899c 100644 --- a/bus/client/hosts.go +++ b/bus/client/hosts.go @@ -8,7 +8,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" ) // Host returns information about a particular host known to the server. @@ -39,7 +38,7 @@ func (c *Client) Hosts(ctx context.Context, opts api.GetHostsOptions) (hosts []a // HostsForScanning returns 'limit' host addresses at given 'offset' which // haven't been scanned after lastScan. -func (c *Client) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) (hosts []hostdb.HostAddress, err error) { +func (c *Client) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) (hosts []api.HostAddress, err error) { values := url.Values{} opts.Apply(values) err = c.c.WithContext(ctx).GET("/hosts/scanning?"+values.Encode(), &hosts) @@ -47,7 +46,7 @@ func (c *Client) HostsForScanning(ctx context.Context, opts api.HostsForScanning } // RecordHostInteraction records an interaction for the supplied host. -func (c *Client) RecordHostScans(ctx context.Context, scans []hostdb.HostScan) (err error) { +func (c *Client) RecordHostScans(ctx context.Context, scans []api.HostScan) (err error) { err = c.c.WithContext(ctx).POST("/hosts/scans", api.HostsScanRequest{ Scans: scans, }, nil) @@ -55,7 +54,7 @@ func (c *Client) RecordHostScans(ctx context.Context, scans []hostdb.HostScan) ( } // RecordHostInteraction records an interaction for the supplied host. -func (c *Client) RecordPriceTables(ctx context.Context, priceTableUpdates []hostdb.PriceTableUpdate) (err error) { +func (c *Client) RecordPriceTables(ctx context.Context, priceTableUpdates []api.HostPriceTableUpdate) (err error) { err = c.c.WithContext(ctx).POST("/hosts/pricetables", api.HostsPriceTablesRequest{ PriceTableUpdates: priceTableUpdates, }, nil) @@ -80,9 +79,11 @@ func (c *Client) ResetLostSectors(ctx context.Context, hostKey types.PublicKey) // SearchHosts returns all hosts that match certain search criteria. func (c *Client) SearchHosts(ctx context.Context, opts api.SearchHostOptions) (hosts []api.Host, err error) { err = c.c.WithContext(ctx).POST("/search/hosts", api.SearchHostsRequest{ + AutopilotID: opts.AutopilotID, Offset: opts.Offset, Limit: opts.Limit, FilterMode: opts.FilterMode, + UsabilityMode: opts.UsabilityMode, AddressContains: opts.AddressContains, KeyIn: opts.KeyIn, }, &hosts) @@ -100,3 +101,10 @@ func (c *Client) UpdateHostBlocklist(ctx context.Context, add, remove []string, err = c.c.WithContext(ctx).PUT("/hosts/blocklist", api.UpdateBlocklistRequest{Add: add, Remove: remove, Clear: clear}) return } + +// UpdateHostCheck updates the host with the most recent check performed by the +// autopilot with given id. +func (c *Client) UpdateHostCheck(ctx context.Context, autopilotID string, hostKey types.PublicKey, hostCheck api.HostCheck) (err error) { + err = c.c.WithContext(ctx).PUT(fmt.Sprintf("/autopilot/%s/host/%s/check", autopilotID, hostKey), hostCheck) + return +} diff --git a/hostdb/hostdb.go b/hostdb/hostdb.go index 1f4c341de..1a957e327 100644 --- a/hostdb/hostdb.go +++ b/hostdb/hostdb.go @@ -4,8 +4,6 @@ import ( "time" "gitlab.com/NebulousLabs/encoding" - rhpv2 "go.sia.tech/core/rhp/v2" - rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/siad/crypto" "go.sia.tech/siad/modules" @@ -59,72 +57,3 @@ func ForEachAnnouncement(b types.Block, height uint64, fn func(types.PublicKey, } } } - -// Interactions contains metadata about a host's interactions. -type Interactions struct { - TotalScans uint64 `json:"totalScans"` - LastScan time.Time `json:"lastScan"` - LastScanSuccess bool `json:"lastScanSuccess"` - LostSectors uint64 `json:"lostSectors"` - SecondToLastScanSuccess bool `json:"secondToLastScanSuccess"` - Uptime time.Duration `json:"uptime"` - Downtime time.Duration `json:"downtime"` - - SuccessfulInteractions float64 `json:"successfulInteractions"` - FailedInteractions float64 `json:"failedInteractions"` -} - -type HostScan struct { - HostKey types.PublicKey `json:"hostKey"` - Success bool - Timestamp time.Time - Settings rhpv2.HostSettings - PriceTable rhpv3.HostPriceTable -} - -type PriceTableUpdate struct { - HostKey types.PublicKey `json:"hostKey"` - Success bool - Timestamp time.Time - PriceTable HostPriceTable -} - -// HostAddress contains the address of a specific host identified by a public -// key. -type HostAddress struct { - PublicKey types.PublicKey `json:"publicKey"` - NetAddress string `json:"netAddress"` -} - -// A Host pairs a host's public key with a set of interactions. -type Host struct { - KnownSince time.Time `json:"knownSince"` - LastAnnouncement time.Time `json:"lastAnnouncement"` - PublicKey types.PublicKey `json:"publicKey"` - NetAddress string `json:"netAddress"` - PriceTable HostPriceTable `json:"priceTable"` - Settings rhpv2.HostSettings `json:"settings"` - Interactions Interactions `json:"interactions"` - Scanned bool `json:"scanned"` -} - -// A HostPriceTable extends the host price table with its expiry. -type HostPriceTable struct { - rhpv3.HostPriceTable - Expiry time.Time `json:"expiry"` -} - -// IsAnnounced returns whether the host has been announced. -func (h Host) IsAnnounced() bool { - return !h.LastAnnouncement.IsZero() -} - -// IsOnline returns whether a host is considered online. -func (h Host) IsOnline() bool { - if h.Interactions.TotalScans == 0 { - return false - } else if h.Interactions.TotalScans == 1 { - return h.Interactions.LastScanSuccess - } - return h.Interactions.LastScanSuccess || h.Interactions.SecondToLastScanSuccess -} diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 71507a8ff..754bf273e 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -23,7 +23,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/internal/test" "go.sia.tech/renterd/object" "go.sia.tech/renterd/wallet" @@ -166,7 +165,7 @@ func TestNewTestCluster(t *testing.T) { if len(hi.Checks.UnusableReasons) != 0 { t.Fatal("usable hosts don't have any reasons set") } - if reflect.DeepEqual(hi.Host, hostdb.Host{}) { + if reflect.DeepEqual(hi.Host, api.Host{}) { t.Fatal("host wasn't set") } if hi.Host.Settings.Release == "" { @@ -191,7 +190,7 @@ func TestNewTestCluster(t *testing.T) { if len(hi.Checks.UnusableReasons) != 0 { t.Fatal("usable hosts don't have any reasons set") } - if reflect.DeepEqual(hi.Host, hostdb.Host{}) { + if reflect.DeepEqual(hi.Host, api.Host{}) { t.Fatal("host wasn't set") } allHosts[hi.Host.PublicKey] = struct{}{} diff --git a/internal/test/e2e/pruning_test.go b/internal/test/e2e/pruning_test.go index de948c970..7c1a856f1 100644 --- a/internal/test/e2e/pruning_test.go +++ b/internal/test/e2e/pruning_test.go @@ -11,7 +11,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/internal/test" ) @@ -32,10 +31,10 @@ func TestHostPruning(t *testing.T) { now := time.Now() recordFailedInteractions := func(n int, hk types.PublicKey) { t.Helper() - his := make([]hostdb.HostScan, n) + his := make([]api.HostScan, n) for i := 0; i < n; i++ { now = now.Add(time.Hour).Add(time.Minute) // 1m leeway - his[i] = hostdb.HostScan{ + his[i] = api.HostScan{ HostKey: hk, Timestamp: now, Success: false, diff --git a/stores/hostdb.go b/stores/hostdb.go index 0b281a0ec..fa93f85b9 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -41,7 +41,7 @@ var ( ) type ( - // dbHost defines a hostdb.Interaction as persisted in the DB. Deleting a + // dbHost defines a api.Interaction as persisted in the DB. Deleting a // host from the db will cascade the deletion and also delete the // corresponding announcements and interactions with that host. // @@ -265,31 +265,29 @@ func (h dbHost) convert(blocked bool) api.Host { checks[check.DBAutopilot.Identifier] = check.convert() } return api.Host{ - Host: hostdb.Host{ - KnownSince: h.CreatedAt, - LastAnnouncement: h.LastAnnouncement, - NetAddress: h.NetAddress, - Interactions: hostdb.Interactions{ - TotalScans: h.TotalScans, - LastScan: lastScan, - LastScanSuccess: h.LastScanSuccess, - SecondToLastScanSuccess: h.SecondToLastScanSuccess, - Uptime: h.Uptime, - Downtime: h.Downtime, - SuccessfulInteractions: h.SuccessfulInteractions, - FailedInteractions: h.FailedInteractions, - LostSectors: h.LostSectors, - }, - PriceTable: hostdb.HostPriceTable{ - HostPriceTable: h.PriceTable.convert(), - Expiry: h.PriceTableExpiry.Time, - }, - PublicKey: types.PublicKey(h.PublicKey), - Scanned: h.Scanned, - Settings: rhpv2.HostSettings(h.Settings), + KnownSince: h.CreatedAt, + LastAnnouncement: h.LastAnnouncement, + NetAddress: h.NetAddress, + Interactions: api.HostInteractions{ + TotalScans: h.TotalScans, + LastScan: lastScan, + LastScanSuccess: h.LastScanSuccess, + SecondToLastScanSuccess: h.SecondToLastScanSuccess, + Uptime: h.Uptime, + Downtime: h.Downtime, + SuccessfulInteractions: h.SuccessfulInteractions, + FailedInteractions: h.FailedInteractions, + LostSectors: h.LostSectors, }, - Blocked: blocked, - Checks: checks, + PriceTable: api.HostPriceTable{ + HostPriceTable: h.PriceTable.convert(), + Expiry: h.PriceTableExpiry.Time, + }, + PublicKey: types.PublicKey(h.PublicKey), + Scanned: h.Scanned, + Settings: rhpv2.HostSettings(h.Settings), + Blocked: blocked, + Checks: checks, } } @@ -430,25 +428,18 @@ func (e *dbBlocklistEntry) blocks(h dbHost) bool { // Host returns information about a host. func (ss *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) { - var h dbHost - - tx := ss.db. - WithContext(ctx). - Where(&dbHost{PublicKey: publicKey(hostKey)}). - Preload("Allowlist"). - Preload("Blocklist"). - Take(&h) - if errors.Is(tx.Error, gorm.ErrRecordNotFound) { + hosts, err := ss.SearchHosts(ctx, "", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", []types.PublicKey{hostKey}, 0, 1) + if err != nil { + return api.Host{}, err + } else if len(hosts) == 0 { return api.Host{}, api.ErrHostNotFound - } else if tx.Error != nil { - return api.Host{}, tx.Error + } else { + return hosts[0], nil } - - return h.convert(ss.isBlocked(h)), nil } func (ss *SQLStore) UpdateHostCheck(ctx context.Context, autopilotID string, hk types.PublicKey, hc api.HostCheck) (err error) { - err = ss.db.Transaction(func(tx *gorm.DB) error { + err = ss.retryTransaction(ctx, (func(tx *gorm.DB) error { // fetch ap id var apID uint if err := tx. @@ -509,12 +500,12 @@ func (ss *SQLStore) UpdateHostCheck(ctx context.Context, autopilotID string, hk GougingUploadErr: hc.Gouging.UploadErr, }). Error - }) + })) return } // HostsForScanning returns the address of hosts for scanning. -func (ss *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]hostdb.HostAddress, error) { +func (ss *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]api.HostAddress, error) { if offset < 0 { return nil, ErrNegativeOffset } @@ -523,7 +514,7 @@ func (ss *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time, PublicKey publicKey `gorm:"unique;index;NOT NULL"` NetAddress string } - var hostAddresses []hostdb.HostAddress + var hostAddresses []api.HostAddress err := ss.db. WithContext(ctx). @@ -534,7 +525,7 @@ func (ss *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time, Order("last_scan ASC"). FindInBatches(&hosts, hostRetrievalBatchSize, func(tx *gorm.DB, batch int) error { for _, h := range hosts { - hostAddresses = append(hostAddresses, hostdb.HostAddress{ + hostAddresses = append(hostAddresses, api.HostAddress{ PublicKey: types.PublicKey(h.PublicKey), NetAddress: h.NetAddress, }) @@ -548,7 +539,7 @@ func (ss *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time, return hostAddresses, err } -func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.Host, error) { +func (ss *SQLStore) SearchHosts(ctx context.Context, autopilotID, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.Host, error) { if offset < 0 { return nil, ErrNegativeOffset } @@ -566,9 +557,11 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains query := ss.db. Model(&dbHost{}). Scopes( + autopilotFilter(autopilotID), hostFilter(filterMode, ss.hasAllowlist(), ss.hasBlocklist()), hostNetAddress(addressContains), hostPublicKey(keyIn), + usabilityFilter(autopilotID, usabilityMode), ) // preload allowlist and blocklist @@ -578,9 +571,6 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains Preload("Blocklist") } - // preload host checks - query = query.Preload("Checks.DBAutopilot") - var hosts []api.Host var fullHosts []dbHost err := query. @@ -607,7 +597,7 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains // Hosts returns non-blocked hosts at given offset and limit. func (ss *SQLStore) Hosts(ctx context.Context, offset, limit int) ([]api.Host, error) { - return ss.SearchHosts(ctx, api.HostFilterModeAllowed, "", nil, offset, limit) + return ss.SearchHosts(ctx, "", api.HostFilterModeAllowed, api.UsabilityFilterModeAll, "", nil, offset, limit) } func (ss *SQLStore) RemoveOfflineHosts(ctx context.Context, minRecentFailures uint64, maxDowntime time.Duration) (removed uint64, err error) { @@ -770,7 +760,7 @@ func (ss *SQLStore) HostBlocklist(ctx context.Context) (blocklist []string, err return } -func (ss *SQLStore) RecordHostScans(ctx context.Context, scans []hostdb.HostScan) error { +func (ss *SQLStore) RecordHostScans(ctx context.Context, scans []api.HostScan) error { if len(scans) == 0 { return nil // nothing to do } @@ -891,7 +881,7 @@ func (ss *SQLStore) RecordHostScans(ctx context.Context, scans []hostdb.HostScan }) } -func (ss *SQLStore) RecordPriceTables(ctx context.Context, priceTableUpdate []hostdb.PriceTableUpdate) error { +func (ss *SQLStore) RecordPriceTables(ctx context.Context, priceTableUpdate []api.HostPriceTableUpdate) error { if len(priceTableUpdate) == 0 { return nil // nothing to do } @@ -1028,6 +1018,17 @@ func hostPublicKey(keyIn []types.PublicKey) func(*gorm.DB) *gorm.DB { } } +// autopilotFilter can be used as a scope to filter host checks based on their +// autopilot +func autopilotFilter(autopilotID string) func(*gorm.DB) *gorm.DB { + return func(db *gorm.DB) *gorm.DB { + if autopilotID == "" { + return db.Preload("Checks.DBAutopilot") + } + return db.Preload("Checks.DBAutopilot", "identifier = ?", autopilotID) + } +} + // hostFilter can be used as a scope to filter hosts based on their filter mode, // returning either all, allowed or blocked hosts. func hostFilter(filterMode string, hasAllowlist, hasBlocklist bool) func(*gorm.DB) *gorm.DB { @@ -1059,6 +1060,26 @@ func hostFilter(filterMode string, hasAllowlist, hasBlocklist bool) func(*gorm.D } } +func usabilityFilter(autopilotID, usabilityMode string) func(*gorm.DB) *gorm.DB { + return func(db *gorm.DB) *gorm.DB { + switch usabilityMode { + case api.UsabilityFilterModeUsable: + db = db. + Joins("INNER JOIN host_checks hc on hc.db_host_id = hosts.id"). + Joins("INNER JOIN autopilots a on a.id = hc.db_autopilot_id AND a.identifier = ?", autopilotID). + Where("hc.usability_blocked = ? AND hc.usability_offline = ? AND hc.usability_low_score = ? AND hc.usability_redundant_ip = ? AND hc.usability_gouging = ? AND hc.usability_not_accepting_contracts = ? AND hc.usability_not_announced = ? AND hc.usability_not_completing_scan = ?", false, false, false, false, false, false, false, false) + case api.UsabilityFilterModeUnusable: + db = db. + Joins("INNER JOIN host_checks hc on hc.db_host_id = hosts.id"). + Joins("INNER JOIN autopilots a on a.id = hc.db_autopilot_id AND a.identifier = ?", autopilotID). + Where("hc.usability_blocked = ? OR hc.usability_offline = ? OR hc.usability_low_score = ? OR hc.usability_redundant_ip = ? OR hc.usability_gouging = ? OR hc.usability_not_accepting_contracts = ? OR hc.usability_not_announced = ? OR hc.usability_not_completing_scan = ?", true, true, true, true, true, true, true, true) + case api.UsabilityFilterModeAll: + // do nothing + } + return db + } +} + func (ss *SQLStore) isBlocked(h dbHost) (blocked bool) { ss.mu.Lock() defer ss.mu.Unlock() diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index ec3bc17be..6adf19968 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -148,7 +148,7 @@ func TestSQLHostDB(t *testing.T) { } func (s *SQLStore) addTestScan(hk types.PublicKey, t time.Time, err error, settings rhpv2.HostSettings) error { - return s.RecordHostScans(context.Background(), []hostdb.HostScan{ + return s.RecordHostScans(context.Background(), []api.HostScan{ { HostKey: hk, Settings: settings, @@ -257,7 +257,7 @@ func TestSearchHosts(t *testing.T) { hk1, hk2, hk3 := hks[0], hks[1], hks[2] // search all hosts - his, err := ss.SearchHosts(context.Background(), api.HostFilterModeAll, "", nil, 0, -1) + his, err := ss.SearchHosts(context.Background(), "", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 0, -1) if err != nil { t.Fatal(err) } else if len(his) != 3 { @@ -265,19 +265,19 @@ func TestSearchHosts(t *testing.T) { } // assert offset & limit are taken into account - his, err = ss.SearchHosts(context.Background(), api.HostFilterModeAll, "", nil, 0, 1) + his, err = ss.SearchHosts(context.Background(), "", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 0, 1) if err != nil { t.Fatal(err) } else if len(his) != 1 { t.Fatal("unexpected") } - his, err = ss.SearchHosts(context.Background(), api.HostFilterModeAll, "", nil, 1, 2) + his, err = ss.SearchHosts(context.Background(), "", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 1, 2) if err != nil { t.Fatal(err) } else if len(his) != 2 { t.Fatal("unexpected") } - his, err = ss.SearchHosts(context.Background(), api.HostFilterModeAll, "", nil, 3, 1) + his, err = ss.SearchHosts(context.Background(), "", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 3, 1) if err != nil { t.Fatal(err) } else if len(his) != 0 { @@ -285,16 +285,16 @@ func TestSearchHosts(t *testing.T) { } // assert address and key filters are taken into account - if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "com:1001", nil, 0, -1); err != nil || len(hosts) != 1 { + if hosts, err := ss.SearchHosts(ctx, "", api.HostFilterModeAll, api.UsabilityFilterModeAll, "com:1001", nil, 0, -1); err != nil || len(hosts) != 1 { t.Fatal("unexpected", len(hosts), err) } - if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "", []types.PublicKey{hk2, hk3}, 0, -1); err != nil || len(hosts) != 2 { + if hosts, err := ss.SearchHosts(ctx, "", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", []types.PublicKey{hk2, hk3}, 0, -1); err != nil || len(hosts) != 2 { t.Fatal("unexpected", len(hosts), err) } - if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "com:1002", []types.PublicKey{hk2, hk3}, 0, -1); err != nil || len(hosts) != 1 { + if hosts, err := ss.SearchHosts(ctx, "", api.HostFilterModeAll, api.UsabilityFilterModeAll, "com:1002", []types.PublicKey{hk2, hk3}, 0, -1); err != nil || len(hosts) != 1 { t.Fatal("unexpected", len(hosts), err) } - if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "com:1002", []types.PublicKey{hk1}, 0, -1); err != nil || len(hosts) != 0 { + if hosts, err := ss.SearchHosts(ctx, "", api.HostFilterModeAll, api.UsabilityFilterModeAll, "com:1002", []types.PublicKey{hk1}, 0, -1); err != nil || len(hosts) != 0 { t.Fatal("unexpected", len(hosts), err) } @@ -303,20 +303,20 @@ func TestSearchHosts(t *testing.T) { if err != nil { t.Fatal(err) } - his, err = ss.SearchHosts(context.Background(), api.HostFilterModeAllowed, "", nil, 0, -1) + his, err = ss.SearchHosts(context.Background(), "", api.HostFilterModeAllowed, api.UsabilityFilterModeAll, "", nil, 0, -1) if err != nil { t.Fatal(err) } else if len(his) != 2 { t.Fatal("unexpected") - } else if his[0].Host.PublicKey != (types.PublicKey{2}) || his[1].Host.PublicKey != (types.PublicKey{3}) { - t.Fatal("unexpected", his[0].Host.PublicKey, his[1].Host.PublicKey) + } else if his[0].PublicKey != (types.PublicKey{2}) || his[1].PublicKey != (types.PublicKey{3}) { + t.Fatal("unexpected", his[0].PublicKey, his[1].PublicKey) } - his, err = ss.SearchHosts(context.Background(), api.HostFilterModeBlocked, "", nil, 0, -1) + his, err = ss.SearchHosts(context.Background(), "", api.HostFilterModeBlocked, api.UsabilityFilterModeAll, "", nil, 0, -1) if err != nil { t.Fatal(err) } else if len(his) != 1 { t.Fatal("unexpected") - } else if his[0].Host.PublicKey != (types.PublicKey{1}) { + } else if his[0].PublicKey != (types.PublicKey{1}) { t.Fatal("unexpected", his) } err = ss.UpdateHostBlocklistEntries(context.Background(), nil, nil, true) @@ -366,7 +366,7 @@ func TestSearchHosts(t *testing.T) { } // fetch all hosts - his, err = ss.SearchHosts(context.Background(), api.HostFilterModeAll, "", nil, 0, -1) + his, err = ss.SearchHosts(context.Background(), "", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 0, -1) if err != nil { t.Fatal(err) } else if cnt != 3 { @@ -382,6 +382,57 @@ func TestSearchHosts(t *testing.T) { t.Fatal("unexpected", c3, ok) } + // assert autopilot filter is taken into account + his, err = ss.SearchHosts(context.Background(), ap1, api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if cnt != 3 { + t.Fatal("unexpected", cnt) + } + + // assert h1 and h2 have the expected checks + if c1, ok := his[0].Checks[ap1]; !ok || c1 != h1c { + t.Fatal("unexpected", c1, ok) + } else if c2, ok := his[1].Checks[ap1]; !ok || c2 != h2c1 { + t.Fatal("unexpected", c2, ok) + } else if _, ok := his[1].Checks[ap2]; ok { + t.Fatal("unexpected") + } + + // assert usability filter is taken into account + h2c1.Usability.RedundantIP = true + err = ss.UpdateHostCheck(context.Background(), ap1, hk2, h2c1) + if err != nil { + t.Fatal(err) + } + his, err = ss.SearchHosts(context.Background(), ap1, api.HostFilterModeAll, api.UsabilityFilterModeUsable, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected", len(his)) + } + + // assert h1 has the expected checks + if c1, ok := his[0].Checks[ap1]; !ok || c1 != h1c { + t.Fatal("unexpected", c1, ok) + } + + his, err = ss.SearchHosts(context.Background(), ap1, api.HostFilterModeAll, api.UsabilityFilterModeUnusable, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected", len(his)) + } else if his[0].PublicKey != hk2 { + t.Fatal("unexpected") + } + + // assert only ap1 check is there + if _, ok := his[0].Checks[ap1]; !ok { + t.Fatal("unexpected") + } else if _, ok := his[0].Checks[ap2]; ok { + t.Fatal("unexpected") + } + // assert cascade delete on host err = ss.db.Exec("DELETE FROM hosts WHERE public_key = ?", publicKey(types.PublicKey{1})).Error if err != nil { @@ -425,7 +476,7 @@ func TestRecordScan(t *testing.T) { if err != nil { t.Fatal(err) } - if host.Interactions != (hostdb.Interactions{}) { + if host.Interactions != (api.HostInteractions{}) { t.Fatal("mismatch") } if host.Settings != (rhpv2.HostSettings{}) { @@ -444,7 +495,7 @@ func TestRecordScan(t *testing.T) { // Record a scan. firstScanTime := time.Now().UTC() settings := rhpv2.HostSettings{NetAddress: "host.com"} - if err := ss.RecordHostScans(ctx, []hostdb.HostScan{newTestScan(hk, firstScanTime, settings, true)}); err != nil { + if err := ss.RecordHostScans(ctx, []api.HostScan{newTestScan(hk, firstScanTime, settings, true)}); err != nil { t.Fatal(err) } host, err = ss.Host(ctx, hk) @@ -459,7 +510,7 @@ func TestRecordScan(t *testing.T) { t.Fatal("wrong time") } host.Interactions.LastScan = time.Time{} - if expected := (hostdb.Interactions{ + if expected := (api.HostInteractions{ TotalScans: 1, LastScan: time.Time{}, LastScanSuccess: true, @@ -477,7 +528,7 @@ func TestRecordScan(t *testing.T) { // Record another scan 1 hour after the previous one. secondScanTime := firstScanTime.Add(time.Hour) - if err := ss.RecordHostScans(ctx, []hostdb.HostScan{newTestScan(hk, secondScanTime, settings, true)}); err != nil { + if err := ss.RecordHostScans(ctx, []api.HostScan{newTestScan(hk, secondScanTime, settings, true)}); err != nil { t.Fatal(err) } host, err = ss.Host(ctx, hk) @@ -489,7 +540,7 @@ func TestRecordScan(t *testing.T) { } host.Interactions.LastScan = time.Time{} uptime += secondScanTime.Sub(firstScanTime) - if host.Interactions != (hostdb.Interactions{ + if host.Interactions != (api.HostInteractions{ TotalScans: 2, LastScan: time.Time{}, LastScanSuccess: true, @@ -504,7 +555,7 @@ func TestRecordScan(t *testing.T) { // Record another scan 2 hours after the second one. This time it fails. thirdScanTime := secondScanTime.Add(2 * time.Hour) - if err := ss.RecordHostScans(ctx, []hostdb.HostScan{newTestScan(hk, thirdScanTime, settings, false)}); err != nil { + if err := ss.RecordHostScans(ctx, []api.HostScan{newTestScan(hk, thirdScanTime, settings, false)}); err != nil { t.Fatal(err) } host, err = ss.Host(ctx, hk) @@ -516,7 +567,7 @@ func TestRecordScan(t *testing.T) { } host.Interactions.LastScan = time.Time{} downtime += thirdScanTime.Sub(secondScanTime) - if host.Interactions != (hostdb.Interactions{ + if host.Interactions != (api.HostInteractions{ TotalScans: 3, LastScan: time.Time{}, LastScanSuccess: false, @@ -566,7 +617,7 @@ func TestRemoveHosts(t *testing.T) { hi2 := newTestScan(hk, t2, rhpv2.HostSettings{NetAddress: "host.com"}, false) // record interactions - if err := ss.RecordHostScans(context.Background(), []hostdb.HostScan{hi1, hi2}); err != nil { + if err := ss.RecordHostScans(context.Background(), []api.HostScan{hi1, hi2}); err != nil { t.Fatal(err) } @@ -594,7 +645,7 @@ func TestRemoveHosts(t *testing.T) { // record interactions t3 := now.Add(-time.Minute * 60) // 1 hour ago (60min downtime) hi3 := newTestScan(hk, t3, rhpv2.HostSettings{NetAddress: "host.com"}, false) - if err := ss.RecordHostScans(context.Background(), []hostdb.HostScan{hi3}); err != nil { + if err := ss.RecordHostScans(context.Background(), []api.HostScan{hi3}); err != nil { t.Fatal(err) } @@ -812,21 +863,21 @@ func TestSQLHostAllowlist(t *testing.T) { assertSearch := func(total, allowed, blocked int) error { t.Helper() - hosts, err := ss.SearchHosts(context.Background(), api.HostFilterModeAll, "", nil, 0, -1) + hosts, err := ss.SearchHosts(context.Background(), "", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 0, -1) if err != nil { return err } if len(hosts) != total { return fmt.Errorf("invalid number of hosts: %v", len(hosts)) } - hosts, err = ss.SearchHosts(context.Background(), api.HostFilterModeAllowed, "", nil, 0, -1) + hosts, err = ss.SearchHosts(context.Background(), "", api.HostFilterModeAllowed, api.UsabilityFilterModeAll, "", nil, 0, -1) if err != nil { return err } if len(hosts) != allowed { return fmt.Errorf("invalid number of hosts: %v", len(hosts)) } - hosts, err = ss.SearchHosts(context.Background(), api.HostFilterModeBlocked, "", nil, 0, -1) + hosts, err = ss.SearchHosts(context.Background(), "", api.HostFilterModeBlocked, api.UsabilityFilterModeAll, "", nil, 0, -1) if err != nil { return err } @@ -1248,8 +1299,8 @@ func hostByPubKey(tx *gorm.DB, hostKey types.PublicKey) (dbHost, error) { } // newTestScan returns a host interaction with given parameters. -func newTestScan(hk types.PublicKey, scanTime time.Time, settings rhpv2.HostSettings, success bool) hostdb.HostScan { - return hostdb.HostScan{ +func newTestScan(hk types.PublicKey, scanTime time.Time, settings rhpv2.HostSettings, success bool) api.HostScan { + return api.HostScan{ HostKey: hk, Success: success, Timestamp: scanTime, @@ -1310,14 +1361,14 @@ func newTestHostCheck() api.HostCheck { Prices: .7, }, Usability: api.HostUsabilityBreakdown{ - Blocked: true, - Offline: true, - LowScore: true, - RedundantIP: true, - Gouging: true, - NotAcceptingContracts: true, - NotAnnounced: true, - NotCompletingScan: true, + Blocked: false, + Offline: false, + LowScore: false, + RedundantIP: false, + Gouging: false, + NotAcceptingContracts: false, + NotAnnounced: false, + NotCompletingScan: false, }, } } diff --git a/worker/client/rhp.go b/worker/client/rhp.go index c71ddd1dc..ec7e10b3d 100644 --- a/worker/client/rhp.go +++ b/worker/client/rhp.go @@ -8,7 +8,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" rhpv2 "go.sia.tech/core/rhp/v2" ) @@ -53,7 +52,7 @@ func (c *Client) RHPFund(ctx context.Context, contractID types.FileContractID, h } // RHPPriceTable fetches a price table for a host. -func (c *Client) RHPPriceTable(ctx context.Context, hostKey types.PublicKey, siamuxAddr string, timeout time.Duration) (pt hostdb.HostPriceTable, err error) { +func (c *Client) RHPPriceTable(ctx context.Context, hostKey types.PublicKey, siamuxAddr string, timeout time.Duration) (pt api.HostPriceTable, err error) { req := api.RHPPriceTableRequest{ HostKey: hostKey, SiamuxAddr: siamuxAddr, diff --git a/worker/host.go b/worker/host.go index cd29572cc..4f4e97496 100644 --- a/worker/host.go +++ b/worker/host.go @@ -12,7 +12,6 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.uber.org/zap" ) @@ -23,7 +22,7 @@ type ( DownloadSector(ctx context.Context, w io.Writer, root types.Hash256, offset, length uint32, overpay bool) error UploadSector(ctx context.Context, sectorRoot types.Hash256, sector *[rhpv2.SectorSize]byte, rev types.FileContractRevision) error - FetchPriceTable(ctx context.Context, rev *types.FileContractRevision) (hpt hostdb.HostPriceTable, err error) + FetchPriceTable(ctx context.Context, rev *types.FileContractRevision) (hpt api.HostPriceTable, err error) FetchRevision(ctx context.Context, fetchTimeout time.Duration) (types.FileContractRevision, error) FundAccount(ctx context.Context, balance types.Currency, rev *types.FileContractRevision) error @@ -187,12 +186,12 @@ func (h *host) RenewContract(ctx context.Context, rrr api.RHPRenewRequest) (_ rh return rev, txnSet, contractPrice, renewErr } -func (h *host) FetchPriceTable(ctx context.Context, rev *types.FileContractRevision) (hpt hostdb.HostPriceTable, err error) { +func (h *host) FetchPriceTable(ctx context.Context, rev *types.FileContractRevision) (hpt api.HostPriceTable, err error) { // fetchPT is a helper function that performs the RPC given a payment function - fetchPT := func(paymentFn PriceTablePaymentFunc) (hpt hostdb.HostPriceTable, err error) { + fetchPT := func(paymentFn PriceTablePaymentFunc) (hpt api.HostPriceTable, err error) { err = h.transportPool.withTransportV3(ctx, h.hk, h.siamuxAddr, func(ctx context.Context, t *transportV3) (err error) { hpt, err = RPCPriceTable(ctx, t, paymentFn) - h.bus.RecordPriceTables(ctx, []hostdb.PriceTableUpdate{ + h.bus.RecordPriceTables(ctx, []api.HostPriceTableUpdate{ { HostKey: h.hk, Success: isSuccessfulInteraction(err), diff --git a/worker/host_test.go b/worker/host_test.go index a993c12e1..3d124e9aa 100644 --- a/worker/host_test.go +++ b/worker/host_test.go @@ -13,7 +13,6 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/internal/test" "lukechampine.com/frand" ) @@ -22,7 +21,7 @@ type ( testHost struct { *hostMock *contractMock - hptFn func() hostdb.HostPriceTable + hptFn func() api.HostPriceTable } testHostManager struct { @@ -57,7 +56,7 @@ func newTestHost(h *hostMock, c *contractMock) *testHost { return newTestHostCustom(h, c, newTestHostPriceTable) } -func newTestHostCustom(h *hostMock, c *contractMock, hptFn func() hostdb.HostPriceTable) *testHost { +func newTestHostCustom(h *hostMock, c *contractMock, hptFn func() api.HostPriceTable) *testHost { return &testHost{ hostMock: h, contractMock: c, @@ -65,11 +64,11 @@ func newTestHostCustom(h *hostMock, c *contractMock, hptFn func() hostdb.HostPri } } -func newTestHostPriceTable() hostdb.HostPriceTable { +func newTestHostPriceTable() api.HostPriceTable { var uid rhpv3.SettingsID frand.Read(uid[:]) - return hostdb.HostPriceTable{ + return api.HostPriceTable{ HostPriceTable: rhpv3.HostPriceTable{UID: uid, HostBlockHeight: 100, Validity: time.Minute}, Expiry: time.Now().Add(time.Minute), } @@ -103,7 +102,7 @@ func (h *testHost) FetchRevision(ctx context.Context, fetchTimeout time.Duration return rev, nil } -func (h *testHost) FetchPriceTable(ctx context.Context, rev *types.FileContractRevision) (hostdb.HostPriceTable, error) { +func (h *testHost) FetchPriceTable(ctx context.Context, rev *types.FileContractRevision) (api.HostPriceTable, error) { return h.hptFn(), nil } diff --git a/worker/interactions.go b/worker/interactions.go index 2107ae582..34e47953a 100644 --- a/worker/interactions.go +++ b/worker/interactions.go @@ -1,13 +1,13 @@ package worker import ( - "go.sia.tech/renterd/hostdb" + "go.sia.tech/renterd/api" ) type ( HostInteractionRecorder interface { - RecordHostScan(...hostdb.HostScan) - RecordPriceTableUpdate(...hostdb.PriceTableUpdate) + RecordHostScan(...api.HostScan) + RecordPriceTableUpdate(...api.HostPriceTableUpdate) } ) diff --git a/worker/mocks_test.go b/worker/mocks_test.go index 47e0de391..0e45b80df 100644 --- a/worker/mocks_test.go +++ b/worker/mocks_test.go @@ -15,7 +15,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/object" "go.sia.tech/renterd/webhooks" ) @@ -267,10 +266,8 @@ func newHostMock(hk types.PublicKey) *hostMock { return &hostMock{ hk: hk, hi: api.Host{ - Host: hostdb.Host{ - PublicKey: hk, - Scanned: true, - }, + PublicKey: hk, + Scanned: true, }, } } @@ -298,11 +295,11 @@ func (hs *hostStoreMock) Host(ctx context.Context, hostKey types.PublicKey) (api return h.hi, nil } -func (hs *hostStoreMock) RecordHostScans(ctx context.Context, scans []hostdb.HostScan) error { +func (hs *hostStoreMock) RecordHostScans(ctx context.Context, scans []api.HostScan) error { return nil } -func (hs *hostStoreMock) RecordPriceTables(ctx context.Context, priceTableUpdate []hostdb.PriceTableUpdate) error { +func (hs *hostStoreMock) RecordPriceTables(ctx context.Context, priceTableUpdate []api.HostPriceTableUpdate) error { return nil } diff --git a/worker/pricetables.go b/worker/pricetables.go index 1bc2ee009..9ca4b1541 100644 --- a/worker/pricetables.go +++ b/worker/pricetables.go @@ -10,7 +10,7 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" - "go.sia.tech/renterd/hostdb" + "go.sia.tech/renterd/api" "lukechampine.com/frand" ) @@ -47,14 +47,14 @@ type ( hk types.PublicKey mu sync.Mutex - hpt hostdb.HostPriceTable + hpt api.HostPriceTable update *priceTableUpdate } priceTableUpdate struct { err error done chan struct{} - hpt hostdb.HostPriceTable + hpt api.HostPriceTable } ) @@ -75,7 +75,7 @@ func newPriceTables(hm HostManager, hs HostStore) *priceTables { } // fetch returns a price table for the given host -func (pts *priceTables) fetch(ctx context.Context, hk types.PublicKey, rev *types.FileContractRevision) (hostdb.HostPriceTable, error) { +func (pts *priceTables) fetch(ctx context.Context, hk types.PublicKey, rev *types.FileContractRevision) (api.HostPriceTable, error) { pts.mu.Lock() pt, exists := pts.priceTables[hk] if !exists { @@ -105,7 +105,7 @@ func (pt *priceTable) ongoingUpdate() (bool, *priceTableUpdate) { return ongoing, pt.update } -func (p *priceTable) fetch(ctx context.Context, rev *types.FileContractRevision) (hpt hostdb.HostPriceTable, err error) { +func (p *priceTable) fetch(ctx context.Context, rev *types.FileContractRevision) (hpt api.HostPriceTable, err error) { // grab the current price table p.mu.Lock() hpt = p.hpt @@ -115,7 +115,7 @@ func (p *priceTable) fetch(ctx context.Context, rev *types.FileContractRevision) // current price table is considered to gouge on the block height gc, err := GougingCheckerFromContext(ctx, false) if err != nil { - return hostdb.HostPriceTable{}, err + return api.HostPriceTable{}, err } // figure out whether we should update the price table, if not we can return @@ -137,7 +137,7 @@ func (p *priceTable) fetch(ctx context.Context, rev *types.FileContractRevision) } else if ongoing { select { case <-ctx.Done(): - return hostdb.HostPriceTable{}, fmt.Errorf("%w; %w", errPriceTableUpdateTimedOut, ctx.Err()) + return api.HostPriceTable{}, fmt.Errorf("%w; %w", errPriceTableUpdateTimedOut, ctx.Err()) case <-update.done: } return update.hpt, update.err @@ -166,14 +166,14 @@ func (p *priceTable) fetch(ctx context.Context, rev *types.FileContractRevision) // sanity check the host has been scanned before fetching the price table if !host.Scanned { - return hostdb.HostPriceTable{}, fmt.Errorf("host %v was not scanned", p.hk) + return api.HostPriceTable{}, fmt.Errorf("host %v was not scanned", p.hk) } // otherwise fetch it h := p.hm.Host(p.hk, types.FileContractID{}, host.Settings.SiamuxAddr()) hpt, err = h.FetchPriceTable(ctx, rev) if err != nil { - return hostdb.HostPriceTable{}, fmt.Errorf("failed to update pricetable, err %v", err) + return api.HostPriceTable{}, fmt.Errorf("failed to update pricetable, err %v", err) } return diff --git a/worker/pricetables_test.go b/worker/pricetables_test.go index 55b0f7057..22c021ccb 100644 --- a/worker/pricetables_test.go +++ b/worker/pricetables_test.go @@ -7,7 +7,6 @@ import ( "time" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" ) func TestPriceTables(t *testing.T) { @@ -45,7 +44,7 @@ func TestPriceTables(t *testing.T) { // manage the host, make sure fetching the price table blocks fetchPTBlockChan := make(chan struct{}) validPT := newTestHostPriceTable() - hm.addHost(newTestHostCustom(h, c, func() hostdb.HostPriceTable { + hm.addHost(newTestHostCustom(h, c, func() api.HostPriceTable { <-fetchPTBlockChan return validPT })) diff --git a/worker/rhpv3.go b/worker/rhpv3.go index c0404b128..22b75adc3 100644 --- a/worker/rhpv3.go +++ b/worker/rhpv3.go @@ -18,7 +18,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/mux/v1" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/internal/utils" "go.sia.tech/siad/crypto" "go.uber.org/zap" @@ -623,36 +622,36 @@ func processPayment(s *streamV3, payment rhpv3.PaymentMethod) error { type PriceTablePaymentFunc func(pt rhpv3.HostPriceTable) (rhpv3.PaymentMethod, error) // RPCPriceTable calls the UpdatePriceTable RPC. -func RPCPriceTable(ctx context.Context, t *transportV3, paymentFunc PriceTablePaymentFunc) (_ hostdb.HostPriceTable, err error) { +func RPCPriceTable(ctx context.Context, t *transportV3, paymentFunc PriceTablePaymentFunc) (_ api.HostPriceTable, err error) { defer wrapErr(&err, "PriceTable") s, err := t.DialStream(ctx) if err != nil { - return hostdb.HostPriceTable{}, err + return api.HostPriceTable{}, err } defer s.Close() var pt rhpv3.HostPriceTable var ptr rhpv3.RPCUpdatePriceTableResponse if err := s.WriteRequest(rhpv3.RPCUpdatePriceTableID, nil); err != nil { - return hostdb.HostPriceTable{}, fmt.Errorf("couldn't send RPCUpdatePriceTableID: %w", err) + return api.HostPriceTable{}, fmt.Errorf("couldn't send RPCUpdatePriceTableID: %w", err) } else if err := s.ReadResponse(&ptr, maxPriceTableSize); err != nil { - return hostdb.HostPriceTable{}, fmt.Errorf("couldn't read RPCUpdatePriceTableResponse: %w", err) + return api.HostPriceTable{}, fmt.Errorf("couldn't read RPCUpdatePriceTableResponse: %w", err) } else if err := json.Unmarshal(ptr.PriceTableJSON, &pt); err != nil { - return hostdb.HostPriceTable{}, fmt.Errorf("couldn't unmarshal price table: %w", err) + return api.HostPriceTable{}, fmt.Errorf("couldn't unmarshal price table: %w", err) } else if payment, err := paymentFunc(pt); err != nil { - return hostdb.HostPriceTable{}, fmt.Errorf("couldn't create payment: %w", err) + return api.HostPriceTable{}, fmt.Errorf("couldn't create payment: %w", err) } else if payment == nil { - return hostdb.HostPriceTable{ + return api.HostPriceTable{ HostPriceTable: pt, Expiry: time.Now(), }, nil // intended not to pay } else if err := processPayment(s, payment); err != nil { - return hostdb.HostPriceTable{}, fmt.Errorf("couldn't process payment: %w", err) + return api.HostPriceTable{}, fmt.Errorf("couldn't process payment: %w", err) } else if err := s.ReadResponse(&rhpv3.RPCPriceTableResponse{}, 0); err != nil { - return hostdb.HostPriceTable{}, fmt.Errorf("couldn't read RPCPriceTableResponse: %w", err) + return api.HostPriceTable{}, fmt.Errorf("couldn't read RPCPriceTableResponse: %w", err) } else { - return hostdb.HostPriceTable{ + return api.HostPriceTable{ HostPriceTable: pt, Expiry: time.Now().Add(pt.Validity), }, nil diff --git a/worker/upload.go b/worker/upload.go index bc419d703..4e82f533e 100644 --- a/worker/upload.go +++ b/worker/upload.go @@ -276,7 +276,6 @@ func (w *worker) threadedUploadPackedSlabs(rs api.RedundancySettings, contractSe // wait for all threads to finish wg.Wait() - return } func (w *worker) tryUploadPackedSlab(ctx context.Context, mem Memory, ps api.PackedSlab, rs api.RedundancySettings, contractSet string, lockPriority int) error { diff --git a/worker/worker.go b/worker/worker.go index 47ba0422c..43450d933 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -24,7 +24,6 @@ import ( "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" "go.sia.tech/renterd/build" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/object" "go.sia.tech/renterd/webhooks" @@ -106,8 +105,8 @@ type ( } HostStore interface { - RecordHostScans(ctx context.Context, scans []hostdb.HostScan) error - RecordPriceTables(ctx context.Context, priceTableUpdate []hostdb.PriceTableUpdate) error + RecordHostScans(ctx context.Context, scans []api.HostScan) error + RecordPriceTables(ctx context.Context, priceTableUpdate []api.HostPriceTableUpdate) error RecordContractSpending(ctx context.Context, records []api.ContractSpendingRecord) error Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) @@ -355,9 +354,9 @@ func (w *worker) rhpPriceTableHandler(jc jape.Context) { // defer interaction recording var err error - var hpt hostdb.HostPriceTable + var hpt api.HostPriceTable defer func() { - w.bus.RecordPriceTables(ctx, []hostdb.PriceTableUpdate{ + w.bus.RecordPriceTables(ctx, []api.HostPriceTableUpdate{ { HostKey: rptr.HostKey, Success: isSuccessfulInteraction(err), @@ -1545,7 +1544,7 @@ func (w *worker) scanHost(ctx context.Context, timeout time.Duration, hostKey ty // record scans that timed out. recordCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - scanErr := w.bus.RecordHostScans(recordCtx, []hostdb.HostScan{ + scanErr := w.bus.RecordHostScans(recordCtx, []api.HostScan{ { HostKey: hostKey, Success: isSuccessfulInteraction(err),