Fetch hosts using the search endpoint

api/host.go

@@ -79,6 +79,7 @@ type (
 		Limit       int
 		Offset      int
 	}
+
 	SearchHostOptions struct {
 		AddressContains string
 		FilterMode      string
@@ -91,7 +92,7 @@ type (
 func DefaultSearchHostOptions() SearchHostOptions {
 	return SearchHostOptions{
 		Limit:      -1,
-		FilterMode: HostFilterModeAll,
+		FilterMode: HostFilterModeAllowed,
 	}
 }
 

autopilot/autopilot.go

@@ -54,10 +54,9 @@ type Bus interface {
 
 	// hostdb
 	Host(ctx context.Context, hostKey types.PublicKey) (hostdb.HostInfo, error)
-	Hosts(ctx context.Context, opts api.GetHostsOptions) ([]hostdb.Host, error)
 	HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error)
 	RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error)
-	SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.Host, error)
+	SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.HostInfo, error)
 
 	// metrics
 	RecordContractSetChurnMetric(ctx context.Context, metrics ...api.ContractSetChurnMetric) error
@@ -196,7 +195,7 @@ func (ap *Autopilot) configHandlerPOST(jc jape.Context) {
 	state := ap.State()
 
 	// fetch hosts
-	hosts, err := ap.bus.Hosts(ctx, api.GetHostsOptions{})
+	hosts, err := ap.bus.SearchHosts(ctx, api.DefaultSearchHostOptions())
 	if jc.Check("failed to get hosts", err) != nil {
 		return
 	}
@@ -735,7 +734,7 @@ func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) {
 	jc.Encode(hosts)
 }
 
-func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.Host) (usables uint64) {
+func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.HostInfo) (usables uint64) {
 	gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow)
 	for _, host := range hosts {
 		usable, _ := isUsableHost(cfg, rs, gc, host, smallestValidScore, 0)
@@ -749,7 +748,7 @@ func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.
 // evaluateConfig evaluates the given configuration and if the gouging settings
 // are too strict for the number of contracts required by 'cfg', it will provide
 // a recommendation on how to loosen it.
-func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.Host) (resp api.ConfigEvaluationResponse) {
+func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.HostInfo) (resp api.ConfigEvaluationResponse) {
 	gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow)
 
 	resp.Hosts = uint64(len(hosts))
@@ -865,7 +864,7 @@ func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Cu
 
 // optimiseGougingSetting tries to optimise one field of the gouging settings to
 // try and hit the target number of contracts.
-func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, hosts []hostdb.Host) bool {
+func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, hosts []hostdb.HostInfo) bool {
 	if cfg.Contracts.Amount == 0 {
 		return true // nothing to do
 	}

autopilot/autopilot_test.go

@@ -14,31 +14,34 @@ import (
 
 func TestOptimiseGougingSetting(t *testing.T) {
 	// create 10 hosts that should all be usable
-	var hosts []hostdb.Host
+	var hosts []hostdb.HostInfo
 	for i := 0; i < 10; i++ {
-		hosts = append(hosts, hostdb.Host{
-			KnownSince: time.Unix(0, 0),
-			PriceTable: hostdb.HostPriceTable{
-				HostPriceTable: rhpv3.HostPriceTable{
-					CollateralCost: types.Siacoins(1),
-					MaxCollateral:  types.Siacoins(1000),
+		hosts = append(hosts, hostdb.HostInfo{
+			Host: hostdb.Host{
+				KnownSince: time.Unix(0, 0),
+				PriceTable: hostdb.HostPriceTable{
+					HostPriceTable: rhpv3.HostPriceTable{
+						CollateralCost: types.Siacoins(1),
+						MaxCollateral:  types.Siacoins(1000),
+					},
 				},
+				Settings: rhpv2.HostSettings{
+					AcceptingContracts: true,
+					Collateral:         types.Siacoins(1),
+					MaxCollateral:      types.Siacoins(1000),
+					Version:            "1.6.0",
+				},
+				Interactions: hostdb.Interactions{
+					Uptime:                  time.Hour * 1000,
+					LastScan:                time.Now(),
+					LastScanSuccess:         true,
+					SecondToLastScanSuccess: true,
+					TotalScans:              100,
+				},
+				LastAnnouncement: time.Unix(0, 0),
+				Scanned:          true,
 			},
-			Settings: rhpv2.HostSettings{
-				AcceptingContracts: true,
-				Collateral:         types.Siacoins(1),
-				MaxCollateral:      types.Siacoins(1000),
-				Version:            "1.6.0",
-			},
-			Interactions: hostdb.Interactions{
-				Uptime:                  time.Hour * 1000,
-				LastScan:                time.Now(),
-				LastScanSuccess:         true,
-				SecondToLastScanSuccess: true,
-				TotalScans:              100,
-			},
-			LastAnnouncement: time.Unix(0, 0),
-			Scanned:          true,
+			Blocked: false,
 		})
 	}
 

autopilot/contractor.go

@@ -249,7 +249,7 @@ func (c *contractor) performContractMaintenance(ctx context.Context, w Worker) (
 	}
 
 	// fetch all hosts
-	hosts, err := c.ap.bus.Hosts(ctx, api.GetHostsOptions{})
+	hosts, err := c.ap.bus.SearchHosts(ctx, api.DefaultSearchHostOptions())
 	if err != nil {
 		return false, err
 	}
@@ -777,7 +777,7 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts
 		host.PriceTable.HostBlockHeight = cs.BlockHeight
 
 		// decide whether the host is still good
-		usable, unusableResult := isUsableHost(state.cfg, state.rs, gc, host.Host, minScore, contract.FileSize())
+		usable, unusableResult := isUsableHost(state.cfg, state.rs, gc, host, minScore, contract.FileSize())
 		if !usable {
 			reasons := unusableResult.reasons()
 			toStopUsing[fcid] = strings.Join(reasons, ",")
@@ -1297,7 +1297,7 @@ func (c *contractor) calculateMinScore(candidates []scoredHost, numContracts uin
 	return minScore
 }
 
-func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.Host, usedHosts map[types.PublicKey]struct{}, storedData map[types.PublicKey]uint64, minScore float64) ([]scoredHost, unusableHostResult, error) {
+func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.HostInfo, usedHosts map[types.PublicKey]struct{}, storedData map[types.PublicKey]uint64, minScore float64) ([]scoredHost, unusableHostResult, error) {
 	start := time.Now()
 
 	// fetch consensus state
@@ -1311,7 +1311,7 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.Host, us
 	gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow)
 
 	// select unused hosts that passed a scan
-	var unused []hostdb.Host
+	var unused []hostdb.HostInfo
 	var excluded, notcompletedscan int
 	for _, h := range hosts {
 		// filter out used hosts
@@ -1348,7 +1348,7 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.Host, us
 		h.PriceTable.HostBlockHeight = cs.BlockHeight
 		usable, result := isUsableHost(state.cfg, state.rs, gc, h, minScore, storedData[h.PublicKey])
 		if usable {
-			candidates = append(candidates, scoredHost{h, result.scoreBreakdown.Score()})
+			candidates = append(candidates, scoredHost{h.Host, result.scoreBreakdown.Score()})
 			continue
 		}
 

autopilot/hostfilter.go

@@ -176,15 +176,18 @@ func (u *unusableHostResult) keysAndValues() []interface{} {
 
 // isUsableHost returns whether the given host is usable along with a list of
 // reasons why it was deemed unusable.
-func isUsableHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.GougingChecker, h hostdb.Host, minScore float64, storedData uint64) (bool, unusableHostResult) {
+func isUsableHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.GougingChecker, h hostdb.HostInfo, minScore float64, storedData uint64) (bool, unusableHostResult) {
 	if rs.Validate() != nil {
 		panic("invalid redundancy settings were supplied - developer error")
 	}
 
 	var errs []error
+	if h.Blocked {
+		errs = append(errs, errHostBlocked)
+	}
+
 	var gougingBreakdown api.HostGougingBreakdown
 	var scoreBreakdown api.HostScoreBreakdown
-
 	if !h.IsAnnounced() {
 		errs = append(errs, errHostNotAnnounced)
 	} else if !h.Scanned {
@@ -211,7 +214,7 @@ func isUsableHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.
 			// not gouging, this because the core package does not have overflow
 			// checks in its cost calculations needed to calculate the period
 			// cost
-			scoreBreakdown = hostScore(cfg, h, storedData, rs.Redundancy())
+			scoreBreakdown = hostScore(cfg, h.Host, storedData, rs.Redundancy())
 			if scoreBreakdown.Score() < minScore {
 				errs = append(errs, fmt.Errorf("%w: (%s): %v < %v", errLowScore, scoreBreakdown.String(), scoreBreakdown.Score(), minScore))
 			}

autopilot/hostinfo.go

@@ -53,7 +53,7 @@ func (c *contractor) HostInfo(ctx context.Context, hostKey types.PublicKey) (api
 	// ignore the pricetable's HostBlockHeight by setting it to our own blockheight
 	host.Host.PriceTable.HostBlockHeight = cs.BlockHeight
 
-	isUsable, unusableResult := isUsableHost(state.cfg, rs, gc, host.Host, minScore, storedData)
+	isUsable, unusableResult := isUsableHost(state.cfg, rs, gc, host, minScore, storedData)
 	return api.HostHandlerResponse{
 		Host: host.Host,
 		Checks: &api.HostHandlerResponseChecks{
@@ -67,7 +67,7 @@ func (c *contractor) HostInfo(ctx context.Context, hostKey types.PublicKey) (api
 	}, nil
 }
 
-func (c *contractor) hostInfoFromCache(ctx context.Context, host hostdb.Host) (hi hostInfo, found bool) {
+func (c *contractor) hostInfoFromCache(ctx context.Context, host hostdb.HostInfo) (hi hostInfo, found bool) {
 	// grab host details from cache
 	c.mu.Lock()
 	hi, found = c.cachedHostInfo[host.PublicKey]
@@ -157,7 +157,7 @@ func (c *contractor) HostInfos(ctx context.Context, filterMode, usabilityMode, a
 				// set IsChecked = false.
 				if usabilityMode == api.UsabilityFilterModeAll {
 					hostInfos = append(hostInfos, api.HostHandlerResponse{
-						Host: host,
+						Host: host.Host,
 					})
 					if wanted > 0 && len(hostInfos) == wanted {
 						return hostInfos, nil // we're done.
@@ -170,7 +170,7 @@ func (c *contractor) HostInfos(ctx context.Context, filterMode, usabilityMode, a
 				continue
 			}
 			hostInfos = append(hostInfos, api.HostHandlerResponse{
-				Host: host,
+				Host: host.Host,
 				Checks: &api.HostHandlerResponseChecks{
 					Gouging:          hi.UnusableResult.gougingBreakdown.Gouging(),
 					GougingBreakdown: hi.UnusableResult.gougingBreakdown,

autopilot/scanner.go

@@ -31,7 +31,7 @@ type (
 		// a bit, we currently use inline interfaces to avoid having to update the
 		// scanner tests with every interface change
 		bus interface {
-			Hosts(ctx context.Context, opts api.GetHostsOptions) ([]hostdb.Host, error)
+			SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.HostInfo, error)
 			HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error)
 			RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error)
 		}

autopilot/scanner_test.go

@@ -19,7 +19,7 @@ type mockBus struct {
 	reqs  []string
 }
 
-func (b *mockBus) Hosts(ctx context.Context, opts api.GetHostsOptions) ([]hostdb.Host, error) {
+func (b *mockBus) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.HostInfo, error) {
 	b.reqs = append(b.reqs, fmt.Sprintf("%d-%d", opts.Offset, opts.Offset+opts.Limit))
 
 	start := opts.Offset
@@ -32,11 +32,15 @@ func (b *mockBus) Hosts(ctx context.Context, opts api.GetHostsOptions) ([]hostdb
 		end = len(b.hosts)
 	}
 
-	return b.hosts[start:end], nil
+	his := make([]hostdb.HostInfo, len(b.hosts[start:end]))
+	for i, h := range b.hosts[start:end] {
+		his[i] = hostdb.HostInfo{Host: h}
+	}
+	return his, nil
 }
 
 func (b *mockBus) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error) {
-	hosts, err := b.Hosts(ctx, api.GetHostsOptions{
+	hosts, err := b.SearchHosts(ctx, api.SearchHostOptions{
 		Offset: opts.Offset,
 		Limit:  opts.Limit,
 	})

bus/bus.go

@@ -92,13 +92,12 @@ type (
 	// A HostDB stores information about hosts.
 	HostDB interface {
 		Host(ctx context.Context, hostKey types.PublicKey) (hostdb.HostInfo, error)
-		Hosts(ctx context.Context, offset, limit int) ([]hostdb.Host, error)
 		HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]hostdb.HostAddress, error)
 		RecordHostScans(ctx context.Context, scans []hostdb.HostScan) error
 		RecordPriceTables(ctx context.Context, priceTableUpdate []hostdb.PriceTableUpdate) error
 		RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error)
 		ResetLostSectors(ctx context.Context, hk types.PublicKey) error
-		SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]hostdb.Host, error)
+		SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]hostdb.HostInfo, error)
 
 		HostAllowlist(ctx context.Context) ([]types.PublicKey, error)
 		HostBlocklist(ctx context.Context) ([]string, error)
@@ -285,7 +284,7 @@ func (b *bus) Handler() http.Handler {
 		"GET    /contract/:id/roots":     b.contractIDRootsHandlerGET,
 		"GET    /contract/:id/size":      b.contractSizeHandlerGET,
 
-		"GET    /hosts":                          b.hostsHandlerGET,
+		"GET    /hosts":                          b.hostsHandlerGETDeprecated,
 		"GET    /hosts/allowlist":                b.hostsAllowlistHandlerGET,
 		"PUT    /hosts/allowlist":                b.hostsAllowlistHandlerPUT,
 		"GET    /hosts/blocklist":                b.hostsBlocklistHandlerGET,
@@ -755,13 +754,15 @@ func (b *bus) walletPendingHandler(jc jape.Context) {
 	jc.Encode(relevant)
 }
 
-func (b *bus) hostsHandlerGET(jc jape.Context) {
+func (b *bus) hostsHandlerGETDeprecated(jc jape.Context) {
 	offset := 0
 	limit := -1
 	if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil {
 		return
 	}
-	hosts, err := b.hdb.Hosts(jc.Request.Context(), offset, limit)
+
+	// fetch hosts
+	hosts, err := b.hdb.SearchHosts(jc.Request.Context(), api.HostFilterModeAllowed, "", nil, offset, limit)
 	if jc.Check(fmt.Sprintf("couldn't fetch hosts %d-%d", offset, offset+limit), err) != nil {
 		return
 	}
@@ -773,6 +774,10 @@ func (b *bus) searchHostsHandlerPOST(jc jape.Context) {
 	if jc.Decode(&req) != nil {
 		return
 	}
+
+	// TODO: on the next major release we should:
+	// - remove api.DefaultSearchHostOptions and set defaults in the handler
+	// - validate the filter mode here and return a 400
 	hosts, err := b.hdb.SearchHosts(jc.Request.Context(), req.FilterMode, req.AddressContains, req.KeyIn, req.Offset, req.Limit)
 	if jc.Check(fmt.Sprintf("couldn't fetch hosts %d-%d", req.Offset, req.Offset+req.Limit), err) != nil {
 		return

bus/client/hosts.go

@@ -30,7 +30,7 @@ func (c *Client) HostBlocklist(ctx context.Context) (blocklist []string, err err
 }
 
 // Hosts returns 'limit' hosts at given 'offset'.
-func (c *Client) Hosts(ctx context.Context, opts api.GetHostsOptions) (hosts []hostdb.Host, err error) {
+func (c *Client) Hosts(ctx context.Context, opts api.GetHostsOptions) (hosts []hostdb.HostInfo, err error) {
 	values := url.Values{}
 	opts.Apply(values)
 	err = c.c.WithContext(ctx).GET("/hosts?"+values.Encode(), &hosts)
@@ -78,7 +78,7 @@ func (c *Client) ResetLostSectors(ctx context.Context, hostKey types.PublicKey)
 }
 
 // SearchHosts returns all hosts that match certain search criteria.
-func (c *Client) SearchHosts(ctx context.Context, opts api.SearchHostOptions) (hosts []hostdb.Host, err error) {
+func (c *Client) SearchHosts(ctx context.Context, opts api.SearchHostOptions) (hosts []hostdb.HostInfo, err error) {
 	err = c.c.WithContext(ctx).POST("/search/hosts", api.SearchHostsRequest{
 		Offset:          opts.Offset,
 		Limit:           opts.Limit,

internal/test/e2e/blocklist_test.go

@@ -23,6 +23,8 @@ func TestBlocklist(t *testing.T) {
 		hosts: 3,
 	})
 	defer cluster.Shutdown()
+
+	// convenience variables
 	b := cluster.Bus
 	tt := cluster.tt
 
@@ -117,7 +119,7 @@ func TestBlocklist(t *testing.T) {
 	}
 
 	// assert we have 4 hosts
-	hosts, err := b.Hosts(context.Background(), api.GetHostsOptions{})
+	hosts, err := b.SearchHosts(context.Background(), api.DefaultSearchHostOptions())
 	tt.OK(err)
 	if len(hosts) != 4 {
 		t.Fatal("unexpected number of hosts", len(hosts))
@@ -142,7 +144,7 @@ func TestBlocklist(t *testing.T) {
 	}
 
 	// assert all others are blocked
-	hosts, err = b.Hosts(context.Background(), api.GetHostsOptions{})
+	hosts, err = b.SearchHosts(context.Background(), api.DefaultSearchHostOptions())
 	tt.OK(err)
 	if len(hosts) != 1 {
 		t.Fatal("unexpected number of hosts", len(hosts))
@@ -152,7 +154,7 @@ func TestBlocklist(t *testing.T) {
 	tt.OK(b.UpdateHostAllowlist(context.Background(), nil, nil, true))
 
 	// assert no hosts are blocked
-	hosts, err = b.Hosts(context.Background(), api.GetHostsOptions{})
+	hosts, err = b.SearchHosts(context.Background(), api.DefaultSearchHostOptions())
 	tt.OK(err)
 	if len(hosts) != 5 {
 		t.Fatal("unexpected number of hosts", len(hosts))

internal/test/e2e/cluster_test.go

@@ -146,7 +146,7 @@ func TestNewTestCluster(t *testing.T) {
 	})
 
 	// Get host info for every host.
-	hosts, err := cluster.Bus.Hosts(context.Background(), api.GetHostsOptions{})
+	hosts, err := cluster.Bus.SearchHosts(context.Background(), api.DefaultSearchHostOptions())
 	tt.OK(err)
 	for _, host := range hosts {
 		hi, err := cluster.Autopilot.HostInfo(host.PublicKey)