Add dbHostCheck entity to the store (#1085)

This PR adds a new entity called `dbHostCheck`. It combines a series of
checks the autopilot performs on a host, and are stored in the `bus`.
The `bus` now returns a new type, simply called `api.Host` which is a
superset of `hostdb.HostInfo`, it adds a map that holds every check for
that host, indexed by the identifier of the autopilot that performed the
check.

api/autopilot.go

@@ -2,8 +2,6 @@ package api
 
 import (
 	"errors"
-	"fmt"
-	"strings"
 
 	"go.sia.tech/core/types"
 	"go.sia.tech/renterd/hostdb"
@@ -136,65 +134,8 @@ type (
 		Usable           bool                 `json:"usable"`
 		UnusableReasons  []string             `json:"unusableReasons"`
 	}
-
-	HostGougingBreakdown struct {
-		ContractErr string `json:"contractErr"`
-		DownloadErr string `json:"downloadErr"`
-		GougingErr  string `json:"gougingErr"`
-		PruneErr    string `json:"pruneErr"`
-		UploadErr   string `json:"uploadErr"`
-	}
-
-	HostScoreBreakdown struct {
-		Age              float64 `json:"age"`
-		Collateral       float64 `json:"collateral"`
-		Interactions     float64 `json:"interactions"`
-		StorageRemaining float64 `json:"storageRemaining"`
-		Uptime           float64 `json:"uptime"`
-		Version          float64 `json:"version"`
-		Prices           float64 `json:"prices"`
-	}
 )
 
-func (sb HostScoreBreakdown) String() string {
-	return fmt.Sprintf("Age: %v, Col: %v, Int: %v, SR: %v, UT: %v, V: %v, Pr: %v", sb.Age, sb.Collateral, sb.Interactions, sb.StorageRemaining, sb.Uptime, sb.Version, sb.Prices)
-}
-
-func (hgb HostGougingBreakdown) Gouging() bool {
-	for _, err := range []string{
-		hgb.ContractErr,
-		hgb.DownloadErr,
-		hgb.GougingErr,
-		hgb.PruneErr,
-		hgb.UploadErr,
-	} {
-		if err != "" {
-			return true
-		}
-	}
-	return false
-}
-
-func (hgb HostGougingBreakdown) String() string {
-	var reasons []string
-	for _, errStr := range []string{
-		hgb.ContractErr,
-		hgb.DownloadErr,
-		hgb.GougingErr,
-		hgb.PruneErr,
-		hgb.UploadErr,
-	} {
-		if errStr != "" {
-			reasons = append(reasons, errStr)
-		}
-	}
-	return strings.Join(reasons, ";")
-}
-
-func (sb HostScoreBreakdown) Score() float64 {
-	return sb.Age * sb.Collateral * sb.Interactions * sb.StorageRemaining * sb.Uptime * sb.Version * sb.Prices
-}
-
 func (c AutopilotConfig) Validate() error {
 	if c.Hosts.MaxDowntimeHours > 99*365*24 {
 		return ErrMaxDowntimeHoursTooHigh

api/host.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"strings"
 
 	"go.sia.tech/core/types"
 	"go.sia.tech/renterd/hostdb"
@@ -42,6 +43,8 @@ type (
 		MinRecentScanFailures uint64    `json:"minRecentScanFailures"`
 	}
 
+	// SearchHostsRequest is the request type for the /api/bus/search/hosts
+	// endpoint.
 	SearchHostsRequest struct {
 		Offset          int               `json:"offset"`
 		Limit           int               `json:"limit"`
@@ -109,3 +112,85 @@ func (opts HostsForScanningOptions) Apply(values url.Values) {
 		values.Set("lastScan", TimeRFC3339(opts.MaxLastScan).String())
 	}
 }
+
+type (
+	Host struct {
+		hostdb.Host
+		Blocked bool                 `json:"blocked"`
+		Checks  map[string]HostCheck `json:"checks"`
+	}
+
+	HostCheck struct {
+		Gouging   HostGougingBreakdown   `json:"gouging"`
+		Score     HostScoreBreakdown     `json:"score"`
+		Usability HostUsabilityBreakdown `json:"usability"`
+	}
+
+	HostGougingBreakdown struct {
+		ContractErr string `json:"contractErr"`
+		DownloadErr string `json:"downloadErr"`
+		GougingErr  string `json:"gougingErr"`
+		PruneErr    string `json:"pruneErr"`
+		UploadErr   string `json:"uploadErr"`
+	}
+
+	HostScoreBreakdown struct {
+		Age              float64 `json:"age"`
+		Collateral       float64 `json:"collateral"`
+		Interactions     float64 `json:"interactions"`
+		StorageRemaining float64 `json:"storageRemaining"`
+		Uptime           float64 `json:"uptime"`
+		Version          float64 `json:"version"`
+		Prices           float64 `json:"prices"`
+	}
+
+	HostUsabilityBreakdown struct {
+		Blocked               bool `json:"blocked"`
+		Offline               bool `json:"offline"`
+		LowScore              bool `json:"lowScore"`
+		RedundantIP           bool `json:"redundantIP"`
+		Gouging               bool `json:"gouging"`
+		NotAcceptingContracts bool `json:"notAcceptingContracts"`
+		NotAnnounced          bool `json:"notAnnounced"`
+		NotCompletingScan     bool `json:"notCompletingScan"`
+	}
+)
+
+func (sb HostScoreBreakdown) String() string {
+	return fmt.Sprintf("Age: %v, Col: %v, Int: %v, SR: %v, UT: %v, V: %v, Pr: %v", sb.Age, sb.Collateral, sb.Interactions, sb.StorageRemaining, sb.Uptime, sb.Version, sb.Prices)
+}
+
+func (hgb HostGougingBreakdown) Gouging() bool {
+	for _, err := range []string{
+		hgb.ContractErr,
+		hgb.DownloadErr,
+		hgb.GougingErr,
+		hgb.PruneErr,
+		hgb.UploadErr,
+	} {
+		if err != "" {
+			return true
+		}
+	}
+	return false
+}
+
+func (hgb HostGougingBreakdown) String() string {
+	var reasons []string
+	for _, errStr := range []string{
+		hgb.ContractErr,
+		hgb.DownloadErr,
+		hgb.GougingErr,
+		hgb.PruneErr,
+		hgb.UploadErr,
+	} {
+		if errStr != "" {
+			reasons = append(reasons, errStr)
+		}
+	}
+	return strings.Join(reasons, ";")
+}
+
+func (sb HostScoreBreakdown) Score() float64 {
+	return sb.Age * sb.Collateral * sb.Interactions * sb.StorageRemaining * sb.Uptime * sb.Version * sb.Prices
+}

autopilot/autopilot.go

@@ -53,10 +53,10 @@ type Bus interface {
 	PrunableData(ctx context.Context) (prunableData api.ContractsPrunableDataResponse, err error)
 
 	// hostdb
-	Host(ctx context.Context, hostKey types.PublicKey) (hostdb.HostInfo, error)
+	Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error)
 	HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error)
 	RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error)
-	SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.HostInfo, error)
+	SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.Host, error)
 
 	// metrics
 	RecordContractSetChurnMetric(ctx context.Context, metrics ...api.ContractSetChurnMetric) error
@@ -737,7 +737,7 @@ func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) {
 	jc.Encode(hosts)
 }
 
-func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.HostInfo) (usables uint64) {
+func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.Host) (usables uint64) {
 	gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow)
 	for _, host := range hosts {
 		usable, _ := isUsableHost(cfg, rs, gc, host, smallestValidScore, 0)
@@ -751,7 +751,7 @@ func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.
 // evaluateConfig evaluates the given configuration and if the gouging settings
 // are too strict for the number of contracts required by 'cfg', it will provide
 // a recommendation on how to loosen it.
-func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.HostInfo) (resp api.ConfigEvaluationResponse) {
+func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.Host) (resp api.ConfigEvaluationResponse) {
 	gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow)
 
 	resp.Hosts = uint64(len(hosts))
@@ -866,7 +866,7 @@ func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Cu
 
 // optimiseGougingSetting tries to optimise one field of the gouging settings to
 // try and hit the target number of contracts.
-func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, hosts []hostdb.HostInfo) bool {
+func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, hosts []api.Host) bool {
 	if cfg.Contracts.Amount == 0 {
 		return true // nothing to do
 	}

autopilot/autopilot_test.go

@@ -14,9 +14,10 @@ import (
 
 func TestOptimiseGougingSetting(t *testing.T) {
 	// create 10 hosts that should all be usable
-	var hosts []hostdb.HostInfo
+	var hosts []api.Host
 	for i := 0; i < 10; i++ {
-		hosts = append(hosts, hostdb.HostInfo{
+		hosts = append(hosts, api.Host{
+
 			Host: hostdb.Host{
 				KnownSince: time.Unix(0, 0),
 				PriceTable: hostdb.HostPriceTable{
@@ -42,6 +43,7 @@ func TestOptimiseGougingSetting(t *testing.T) {
 				Scanned:          true,
 			},
 			Blocked: false,
+			Checks:  nil,
 		})
 	}
 

autopilot/client.go

@@ -40,7 +40,7 @@ func (c *Client) HostInfo(hostKey types.PublicKey) (resp api.HostHandlerResponse
 }
 
 // HostInfo returns information about all hosts.
-func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode string, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostHandlerResponse, err error) {
+func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostHandlerResponse, err error) {
 	err = c.c.POST("/hosts", api.SearchHostsRequest{
 		Offset:          offset,
 		Limit:           limit,

autopilot/contractor.go

@@ -1297,7 +1297,7 @@ func (c *contractor) calculateMinScore(candidates []scoredHost, numContracts uin
 	return minScore
 }
 
-func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.HostInfo, usedHosts map[types.PublicKey]struct{}, storedData map[types.PublicKey]uint64, minScore float64) ([]scoredHost, unusableHostResult, error) {
+func (c *contractor) candidateHosts(ctx context.Context, hosts []api.Host, usedHosts map[types.PublicKey]struct{}, storedData map[types.PublicKey]uint64, minScore float64) ([]scoredHost, unusableHostResult, error) {
 	start := time.Now()
 
 	// fetch consensus state
@@ -1311,7 +1311,7 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.HostInfo
 	gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow)
 
 	// select unused hosts that passed a scan
-	var unused []hostdb.HostInfo
+	var unused []api.Host
 	var excluded, notcompletedscan int
 	for _, h := range hosts {
 		// filter out used hosts
@@ -1612,7 +1612,7 @@ func (c *contractor) tryPerformPruning(wp *workerPool) {
 	}()
 }
 
-func (c *contractor) hostForContract(ctx context.Context, fcid types.FileContractID) (host hostdb.HostInfo, metadata api.ContractMetadata, err error) {
+func (c *contractor) hostForContract(ctx context.Context, fcid types.FileContractID) (host api.Host, metadata api.ContractMetadata, err error) {
 	// fetch the contract
 	metadata, err = c.ap.bus.Contract(ctx, fcid)
 	if err != nil {

autopilot/hostfilter.go

@@ -11,7 +11,6 @@ import (
 	rhpv3 "go.sia.tech/core/rhp/v3"
 	"go.sia.tech/core/types"
 	"go.sia.tech/renterd/api"
-	"go.sia.tech/renterd/hostdb"
 	"go.sia.tech/renterd/worker"
 )
 
@@ -176,7 +175,7 @@ func (u *unusableHostResult) keysAndValues() []interface{} {
 
 // isUsableHost returns whether the given host is usable along with a list of
 // reasons why it was deemed unusable.
-func isUsableHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.GougingChecker, h hostdb.HostInfo, minScore float64, storedData uint64) (bool, unusableHostResult) {
+func isUsableHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.GougingChecker, h api.Host, minScore float64, storedData uint64) (bool, unusableHostResult) {
 	if rs.Validate() != nil {
 		panic("invalid redundancy settings were supplied - developer error")
 	}

autopilot/hostinfo.go

@@ -6,7 +6,6 @@ import (
 
 	"go.sia.tech/core/types"
 	"go.sia.tech/renterd/api"
-	"go.sia.tech/renterd/hostdb"
 	"go.sia.tech/renterd/worker"
 )
 
@@ -67,7 +66,7 @@ func (c *contractor) HostInfo(ctx context.Context, hostKey types.PublicKey) (api
 	}, nil
 }
 
-func (c *contractor) hostInfoFromCache(ctx context.Context, host hostdb.HostInfo) (hi hostInfo, found bool) {
+func (c *contractor) hostInfoFromCache(ctx context.Context, host api.Host) (hi hostInfo, found bool) {
 	// grab host details from cache
 	c.mu.Lock()
 	hi, found = c.cachedHostInfo[host.PublicKey]

autopilot/scanner.go

@@ -31,7 +31,7 @@ type (
 		// a bit, we currently use inline interfaces to avoid having to update the
 		// scanner tests with every interface change
 		bus interface {
-			SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.HostInfo, error)
+			SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.Host, error)
 			HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error)
 			RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error)
 		}

autopilot/scanner_test.go

@@ -19,7 +19,7 @@ type mockBus struct {
 	reqs  []string
 }
 
-func (b *mockBus) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.HostInfo, error) {
+func (b *mockBus) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.Host, error) {
 	b.reqs = append(b.reqs, fmt.Sprintf("%d-%d", opts.Offset, opts.Offset+opts.Limit))
 
 	start := opts.Offset
@@ -32,11 +32,11 @@ func (b *mockBus) SearchHosts(ctx context.Context, opts api.SearchHostOptions) (
 		end = len(b.hosts)
 	}
 
-	his := make([]hostdb.HostInfo, len(b.hosts[start:end]))
+	hosts := make([]api.Host, len(b.hosts[start:end]))
 	for i, h := range b.hosts[start:end] {
-		his[i] = hostdb.HostInfo{Host: h}
+		hosts[i] = api.Host{Host: h}
 	}
-	return his, nil
+	return hosts, nil
 }
 
 func (b *mockBus) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error) {

bus/bus.go

@@ -91,13 +91,13 @@ type (
 
 	// A HostDB stores information about hosts.
 	HostDB interface {
-		Host(ctx context.Context, hostKey types.PublicKey) (hostdb.HostInfo, error)
+		Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error)
 		HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]hostdb.HostAddress, error)
 		RecordHostScans(ctx context.Context, scans []hostdb.HostScan) error
 		RecordPriceTables(ctx context.Context, priceTableUpdate []hostdb.PriceTableUpdate) error
 		RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error)
 		ResetLostSectors(ctx context.Context, hk types.PublicKey) error
-		SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]hostdb.HostInfo, error)
+		SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.Host, error)
 
 		HostAllowlist(ctx context.Context) ([]types.PublicKey, error)
 		HostBlocklist(ctx context.Context) ([]string, error)
@@ -775,9 +775,9 @@ func (b *bus) searchHostsHandlerPOST(jc jape.Context) {
 		return
 	}
 
-	// TODO: on the next major release
-	// - set defaults in handler
-	// - validate request params and return 400 if invalid
+	// TODO: on the next major release:
+	// - properly default search params
+	// - properly validate and return 400
 	hosts, err := b.hdb.SearchHosts(jc.Request.Context(), req.FilterMode, req.AddressContains, req.KeyIn, req.Offset, req.Limit)
 	if jc.Check(fmt.Sprintf("couldn't fetch hosts %d-%d", req.Offset, req.Offset+req.Limit), err) != nil {
 		return

bus/client/hosts.go

@@ -12,7 +12,7 @@ import (
 )
 
 // Host returns information about a particular host known to the server.
-func (c *Client) Host(ctx context.Context, hostKey types.PublicKey) (h hostdb.HostInfo, err error) {
+func (c *Client) Host(ctx context.Context, hostKey types.PublicKey) (h api.Host, err error) {
 	err = c.c.WithContext(ctx).GET(fmt.Sprintf("/host/%s", hostKey), &h)
 	return
 }
@@ -30,7 +30,7 @@ func (c *Client) HostBlocklist(ctx context.Context) (blocklist []string, err err
 }
 
 // Hosts returns 'limit' hosts at given 'offset'.
-func (c *Client) Hosts(ctx context.Context, opts api.GetHostsOptions) (hosts []hostdb.HostInfo, err error) {
+func (c *Client) Hosts(ctx context.Context, opts api.GetHostsOptions) (hosts []api.Host, err error) {
 	values := url.Values{}
 	opts.Apply(values)
 	err = c.c.WithContext(ctx).GET("/hosts?"+values.Encode(), &hosts)
@@ -78,7 +78,7 @@ func (c *Client) ResetLostSectors(ctx context.Context, hostKey types.PublicKey)
 }
 
 // SearchHosts returns all hosts that match certain search criteria.
-func (c *Client) SearchHosts(ctx context.Context, opts api.SearchHostOptions) (hosts []hostdb.HostInfo, err error) {
+func (c *Client) SearchHosts(ctx context.Context, opts api.SearchHostOptions) (hosts []api.Host, err error) {
 	err = c.c.WithContext(ctx).POST("/search/hosts", api.SearchHostsRequest{
 		Offset:          opts.Offset,
 		Limit:           opts.Limit,