stores: add autopilot and usability filter to SearchHosts

api/host.go

@@ -24,10 +24,6 @@ var (
 	// ErrHostNotFound is returned when a host can't be retrieved from the
 	// database.
 	ErrHostNotFound = errors.New("host doesn't exist in hostdb")
-
-	// ErrHostInfoNotFound is returned when host info can't be retrieved from
-	// the database.
-	ErrHostInfoNotFound = errors.New("host info doesn't exist in hostdb")
 )
 
 var (
@@ -71,11 +67,8 @@ type (
 		KeyIn           []types.PublicKey `json:"keyIn"`
 	}
 
-	// HostsRequest is the request type for the POST /autopilot/:id/hosts
-	// endpoint.
-	HostsRequest SearchHostsRequest
-
-	// HostResponse is the response type for the /host/:hostkey endpoint.
+	// HostResponse is the response type for the GET
+	// /api/autopilot/host/:hostkey endpoint.
 	HostResponse struct {
 		Host   hostdb.Host `json:"host"`
 		Checks *HostChecks `json:"checks,omitempty"`
@@ -120,6 +113,7 @@ type (
 	}
 
 	SearchHostOptions struct {
+		AutopilotID     string
 		AddressContains string
 		FilterMode      string
 		UsabilityMode   string

autopilot/autopilot.go

@@ -705,6 +705,13 @@ func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) {
 	var req api.SearchHostOptions
 	if jc.Decode(&req) != nil {
 		return
+	} else if req.AutopilotID != "" && req.AutopilotID != ap.id {
+		jc.Error(errors.New("invalid autopilot id"), http.StatusBadRequest)
+		return
+	} else {
+		// TODO: on next major release we should not re-use options between bus
+		// and autopilot API if we don't support all fields in both
+		req.AutopilotID = ap.id
 	}
 
 	// TODO: remove on next major release

autopilot/contractor.go

@@ -713,26 +713,29 @@ func (c *contractor) runContractChecks(ctx context.Context, contracts []api.Cont
 		// fetch host from hostdb
 		host, err := c.ap.bus.Host(ctx, hk)
 		if err != nil {
-			c.logger.Errorw(fmt.Sprintf("missing host, err: %v", err), "hk", hk)
+			c.logger.Warn(fmt.Sprintf("missing host, err: %v", err), "hk", hk)
 			toStopUsing[fcid] = api.ErrUsabilityHostNotFound.Error()
 			notfound++
 			continue
 		}
 
+		// fetch host checks
+		check, ok := host.Checks[c.ap.id]
+		if !ok {
+			// this is only possible due to developer error, if there is no
+			// check the host would have been missing, so we treat it the same
+			c.logger.Warnw("missing host check", "hk", hk)
+			toStopUsing[fcid] = api.ErrUsabilityHostNotFound.Error()
+			continue
+		}
+
 		// if the host is blocked we ignore it, it might be unblocked later
 		if host.Blocked {
 			c.logger.Infow("unusable host", "hk", hk, "fcid", fcid, "reasons", api.ErrUsabilityHostBlocked.Error())
 			toStopUsing[fcid] = api.ErrUsabilityHostBlocked.Error()
 			continue
 		}
 
-		// grab the host check
-		check, ok := host.Checks[c.ap.id]
-		if !ok {
-			c.logger.Errorw("missing host check", "hk", hk)
-			continue
-		}
-
 		// check if the host is still usable
 		if !check.Usability.IsUsable() {
 			reasons := check.Usability.UnusableReasons()

bus/bus.go

@@ -253,9 +253,7 @@ func (b *bus) Handler() http.Handler {
 		"GET    /autopilot/:id": b.autopilotsHandlerGET,
 		"PUT    /autopilot/:id": b.autopilotsHandlerPUT,
 
-		"GET    /autopilot/:id/host/:hostkey": b.autopilotHostHandlerGET,
-		"PUT    /autopilot/:id/host/:hostkey": b.autopilotHostHandlerPUT,
-		"POST   /autopilot/:id/hosts":         b.autopilotHostsHandlerGET,
+		"PUT    /autopilot/:id/host/:hostkey/checks": b.autopilotHostChecksHandlerPUT,
 
 		"GET    /buckets":             b.bucketsHandlerGET,
 		"POST   /buckets":             b.bucketsHandlerPOST,
@@ -780,8 +778,9 @@ func (b *bus) searchHostsHandlerPOST(jc jape.Context) {
 	}
 
 	// TODO: on the next major release:
-	// - properly default search params
-	// - properly validate and return 400
+	// - properly default search params (currenlty no defaults are set)
+	// - properly validate and return 400 (currently validation is done in autopilot and the store)
+
 	hosts, err := b.hdb.SearchHosts(jc.Request.Context(), req.AutopilotID, req.FilterMode, req.UsabilityMode, req.AddressContains, req.KeyIn, req.Offset, req.Limit)
 	if jc.Check(fmt.Sprintf("couldn't fetch hosts %d-%d", req.Offset, req.Offset+req.Limit), err) != nil {
 		return
@@ -1970,27 +1969,7 @@ func (b *bus) autopilotsHandlerPUT(jc jape.Context) {
 	jc.Check("failed to update autopilot", b.as.UpdateAutopilot(jc.Request.Context(), ap))
 }
 
-func (b *bus) autopilotHostHandlerGET(jc jape.Context) {
-	var id string
-	if jc.DecodeParam("id", &id) != nil {
-		return
-	}
-	var hk types.PublicKey
-	if jc.DecodeParam("hostkey", &hk) != nil {
-		return
-	}
-
-	hi, err := b.hdb.Host(jc.Request.Context(), hk)
-	if errors.Is(err, api.ErrAutopilotNotFound) {
-		jc.Error(err, http.StatusNotFound)
-		return
-	} else if jc.Check("failed to fetch host info", err) != nil {
-		return
-	}
-	jc.Encode(hi)
-}
-
-func (b *bus) autopilotHostHandlerPUT(jc jape.Context) {
+func (b *bus) autopilotHostChecksHandlerPUT(jc jape.Context) {
 	var id string
 	if jc.DecodeParam("id", &id) != nil {
 		return
@@ -2013,49 +1992,6 @@ func (b *bus) autopilotHostHandlerPUT(jc jape.Context) {
 	}
 }
 
-func (b *bus) autopilotHostsHandlerGET(jc jape.Context) {
-	var id string
-	if jc.DecodeParam("id", &id) != nil {
-		return
-	}
-	var req api.HostsRequest
-	if jc.Decode(&req) != nil {
-		return
-	}
-
-	// validate filter mode
-	if fm := req.FilterMode; fm != "" {
-		if fm != api.HostFilterModeAll &&
-			fm != api.HostFilterModeAllowed &&
-			fm != api.HostFilterModeBlocked {
-			jc.Error(fmt.Errorf("invalid filter mode: '%v', allowed values are '%s', '%s', '%s'", fm, api.HostFilterModeAll, api.HostFilterModeAllowed, api.HostFilterModeBlocked), http.StatusBadRequest)
-			return
-		}
-	}
-
-	// validate usability mode
-	if um := req.UsabilityMode; um != "" {
-		if um != api.UsabilityFilterModeUsable &&
-			um != api.UsabilityFilterModeUnusable &&
-			um != api.UsabilityFilterModeAll {
-			jc.Error(fmt.Errorf("invalid usability mode: '%v', allowed values are '%s', '%s', '%s'", um, api.UsabilityFilterModeAll, api.UsabilityFilterModeUsable, api.UsabilityFilterModeUnusable), http.StatusBadRequest)
-			return
-		} else if id == "" {
-			jc.Error(errors.New("usability mode requires autopilot id"), http.StatusBadRequest)
-			return
-		}
-	}
-
-	his, err := b.hdb.SearchHosts(jc.Request.Context(), id, req.FilterMode, req.UsabilityMode, req.AddressContains, req.KeyIn, req.Offset, req.Limit)
-	if errors.Is(err, api.ErrAutopilotNotFound) {
-		jc.Error(err, http.StatusNotFound)
-		return
-	} else if jc.Check("failed to fetch host infos", err) != nil {
-		return
-	}
-	jc.Encode(his)
-}
-
 func (b *bus) contractTaxHandlerGET(jc jape.Context) {
 	var payout types.Currency
 	if jc.DecodeParam("payout", (*api.ParamCurrency)(&payout)) != nil {

bus/client/hosts.go

@@ -80,9 +80,11 @@ func (c *Client) ResetLostSectors(ctx context.Context, hostKey types.PublicKey)
 // SearchHosts returns all hosts that match certain search criteria.
 func (c *Client) SearchHosts(ctx context.Context, opts api.SearchHostOptions) (hosts []api.Host, err error) {
 	err = c.c.WithContext(ctx).POST("/search/hosts", api.SearchHostsRequest{
+		AutopilotID:     opts.AutopilotID,
 		Offset:          opts.Offset,
 		Limit:           opts.Limit,
 		FilterMode:      opts.FilterMode,
+		UsabilityMode:   opts.UsabilityMode,
 		AddressContains: opts.AddressContains,
 		KeyIn:           opts.KeyIn,
 	}, &hosts)

stores/hostdb.go

@@ -627,9 +627,11 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, autopilotID, filterMode, us
 	query := ss.db.
 		Model(&dbHost{}).
 		Scopes(
+			autopilotFilter(autopilotID),
 			hostFilter(filterMode, ss.hasAllowlist(), ss.hasBlocklist()),
 			hostNetAddress(addressContains),
 			hostPublicKey(keyIn),
+			usabilityFilter(usabilityMode),
 		)
 
 	// preload allowlist and blocklist
@@ -639,12 +641,23 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, autopilotID, filterMode, us
 			Preload("Blocklist")
 	}
 
-	// preload host checks
-	query = query.Preload("Checks.DBAutopilot")
+	// filter checks
+	if autopilotID != "" {
+		query = query.Preload("Checks.DBAutopilot", "identifier = ?", autopilotID)
+	} else {
+		query = query.Preload("Checks.DBAutopilot")
+	}
+	// query = query.
+	// 	Preload("Checks.DBAutopilot").
+	// 	Scopes(
+	// 		autopilotFilter(autopilotID),
+	// 		usabilityFilter(usabilityMode),
+	// 	)
 
 	var hosts []api.Host
 	var fullHosts []dbHost
 	err := query.
+		Debug().
 		Offset(offset).
 		Limit(limit).
 		FindInBatches(&fullHosts, hostRetrievalBatchSize, func(tx *gorm.DB, batch int) error {
@@ -1089,6 +1102,17 @@ func hostPublicKey(keyIn []types.PublicKey) func(*gorm.DB) *gorm.DB {
 	}
 }
 
+// autopilotFilter can be used as a scope to filter host checks based on their
+// autopilot
+func autopilotFilter(autopilotID string) func(*gorm.DB) *gorm.DB {
+	return func(db *gorm.DB) *gorm.DB {
+		if autopilotID == "" {
+			return db.Preload("Checks.DBAutopilot")
+		}
+		return db.Preload("Checks.DBAutopilot", "identifier = ?", autopilotID)
+	}
+}
+
 // hostFilter can be used as a scope to filter hosts based on their filter mode,
 // returning either all, allowed or blocked hosts.
 func hostFilter(filterMode string, hasAllowlist, hasBlocklist bool) func(*gorm.DB) *gorm.DB {
@@ -1120,6 +1144,20 @@ func hostFilter(filterMode string, hasAllowlist, hasBlocklist bool) func(*gorm.D
 	}
 }
 
+func usabilityFilter(usabilityMode string) func(*gorm.DB) *gorm.DB {
+	return func(db *gorm.DB) *gorm.DB {
+		switch usabilityMode {
+		case api.UsabilityFilterModeUsable:
+			db = db.Preload("Checks", "usability_blocked = ? AND usability_offline = ? AND usability_low_score = ? AND usability_redundant_ip = ? AND usability_gouging = ? AND usability_not_accepting_contracts = ? AND usability_not_announced = ? AND usability_not_completing_scan = ?", false, false, false, false, false, false, false, false)
+		case api.UsabilityFilterModeUnusable:
+			db = db.Preload("Checks", "usability_blocked = ? OR usability_offline = ? OR usability_low_score = ? OR usability_redundant_ip = ? OR usability_gouging = ? OR usability_not_accepting_contracts = ? OR usability_not_announced = ? OR usability_not_completing_scan = ?", true, true, true, true, true, true, true, true)
+		case api.UsabilityFilterModeAll:
+			// do nothing
+		}
+		return db
+	}
+}
+
 func (ss *SQLStore) isBlocked(h dbHost) (blocked bool) {
 	ss.mu.Lock()
 	defer ss.mu.Unlock()

stores/hostdb_test.go

@@ -382,6 +382,61 @@ func TestSearchHosts(t *testing.T) {
 		t.Fatal("unexpected", c3, ok)
 	}
 
+	// assert autopilot filter is taken into account
+	his, err = ss.SearchHosts(context.Background(), ap1, api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 0, -1)
+	if err != nil {
+		t.Fatal(err)
+	} else if cnt != 3 {
+		t.Fatal("unexpected", cnt)
+	}
+
+	// assert h1 and h2 have the expected checks
+	if c1, ok := his[0].Checks[ap1]; !ok || c1 != h1c {
+		t.Fatal("unexpected", c1, ok)
+	} else if c2, ok := his[1].Checks[ap1]; !ok || c2 != h2c1 {
+		t.Fatal("unexpected", c2, ok)
+	} else if _, ok := his[1].Checks[ap2]; ok {
+		t.Fatal("unexpected")
+	}
+
+	// assert usability filter is taken into account
+	h2c1.Usability.RedundantIP = true
+	err = ss.UpdateHostCheck(context.Background(), ap1, hk2, h2c1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	his, err = ss.SearchHosts(context.Background(), ap1, api.HostFilterModeAll, api.UsabilityFilterModeUsable, "", nil, 0, -1)
+	if err != nil {
+		t.Fatal(err)
+	} else if cnt != 3 {
+		t.Fatal("unexpected", cnt)
+	}
+
+	// assert h1 and h2 have the expected checks
+	if c1, ok := his[0].Checks[ap1]; !ok || c1 != h1c {
+		t.Fatal("unexpected", c1, ok)
+	} else if _, ok := his[1].Checks[ap1]; ok {
+		t.Fatal("unexpected", ok)
+	} else if _, ok := his[1].Checks[ap2]; ok {
+		t.Fatal("unexpected")
+	}
+
+	his, err = ss.SearchHosts(context.Background(), ap1, api.HostFilterModeAll, api.UsabilityFilterModeUnusable, "", nil, 0, -1)
+	if err != nil {
+		t.Fatal(err)
+	} else if cnt != 3 {
+		t.Fatal("unexpected", cnt)
+	}
+
+	// assert h1 and h2 have the expected checks
+	if _, ok := his[0].Checks[ap1]; ok {
+		t.Fatal("unexpected")
+	} else if c2, ok := his[1].Checks[ap1]; !ok || c2 != h2c1 {
+		t.Fatal("unexpected", ok)
+	} else if _, ok := his[1].Checks[ap2]; ok {
+		t.Fatal("unexpected")
+	}
+
 	// assert cascade delete on host
 	err = ss.db.Exec("DELETE FROM hosts WHERE public_key = ?", publicKey(types.PublicKey{1})).Error
 	if err != nil {
@@ -1310,14 +1365,14 @@ func newTestHostCheck() api.HostCheck {
 			Prices:           .7,
 		},
 		Usability: api.HostUsabilityBreakdown{
-			Blocked:               true,
-			Offline:               true,
-			LowScore:              true,
-			RedundantIP:           true,
-			Gouging:               true,
-			NotAcceptingContracts: true,
-			NotAnnounced:          true,
-			NotCompletingScan:     true,
+			Blocked:               false,
+			Offline:               false,
+			LowScore:              false,
+			RedundantIP:           false,
+			Gouging:               false,
+			NotAcceptingContracts: false,
+			NotAnnounced:          false,
+			NotCompletingScan:     false,
 		},
 	}
 }