From b8c56488ef95b46f78ede30dc8586040b2ec4d2a Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 18 Mar 2024 17:09:12 +0100 Subject: [PATCH 01/23] stores: add dbHostInfo --- api/autopilot.go | 66 ----- api/host.go | 106 +++++++- bus/bus.go | 67 ++++++ stores/hostdb.go | 226 ++++++++++++++++++ stores/hostdb_test.go | 122 ++++++++++ stores/migrations.go | 6 + .../mysql/main/migration_00007_host_info.sql | 54 +++++ stores/migrations/mysql/main/schema.sql | 55 +++++ .../sqlite/main/migration_00007_host_info.sql | 54 +++++ stores/migrations/sqlite/main/schema.sql | 20 ++ 10 files changed, 703 insertions(+), 73 deletions(-) create mode 100644 stores/migrations/mysql/main/migration_00007_host_info.sql create mode 100644 stores/migrations/sqlite/main/migration_00007_host_info.sql diff --git a/api/autopilot.go b/api/autopilot.go index fdd6c4942..b20696fa9 100644 --- a/api/autopilot.go +++ b/api/autopilot.go @@ -2,8 +2,6 @@ package api import ( "errors" - "fmt" - "strings" "go.sia.tech/core/types" "go.sia.tech/renterd/hostdb" @@ -136,68 +134,4 @@ type ( Usable bool `json:"usable"` UnusableReasons []string `json:"unusableReasons"` } - - HostGougingBreakdown struct { - ContractErr string `json:"contractErr"` - DownloadErr string `json:"downloadErr"` - GougingErr string `json:"gougingErr"` - PruneErr string `json:"pruneErr"` - UploadErr string `json:"uploadErr"` - } - - HostScoreBreakdown struct { - Age float64 `json:"age"` - Collateral float64 `json:"collateral"` - Interactions float64 `json:"interactions"` - StorageRemaining float64 `json:"storageRemaining"` - Uptime float64 `json:"uptime"` - Version float64 `json:"version"` - Prices float64 `json:"prices"` - } ) - -func (sb HostScoreBreakdown) String() string { - return fmt.Sprintf("Age: %v, Col: %v, Int: %v, SR: %v, UT: %v, V: %v, Pr: %v", sb.Age, sb.Collateral, sb.Interactions, sb.StorageRemaining, sb.Uptime, sb.Version, sb.Prices) -} - -func (hgb HostGougingBreakdown) Gouging() bool { - for _, err := range []string{ - hgb.ContractErr, - hgb.DownloadErr, - hgb.GougingErr, - hgb.PruneErr, - hgb.UploadErr, - } { - if err != "" { - return true - } - } - return false -} - -func (hgb HostGougingBreakdown) String() string { - var reasons []string - for _, errStr := range []string{ - hgb.ContractErr, - hgb.DownloadErr, - hgb.GougingErr, - hgb.PruneErr, - hgb.UploadErr, - } { - if errStr != "" { - reasons = append(reasons, errStr) - } - } - return strings.Join(reasons, ";") -} - -func (sb HostScoreBreakdown) Score() float64 { - return sb.Age * sb.Collateral * sb.Interactions * sb.StorageRemaining * sb.Uptime * sb.Version * sb.Prices -} - -func (c AutopilotConfig) Validate() error { - if c.Hosts.MaxDowntimeHours > 99*365*24 { - return ErrMaxDowntimeHoursTooHigh - } - return nil -} diff --git a/api/host.go b/api/host.go index aea80a9fe..05905d5b3 100644 --- a/api/host.go +++ b/api/host.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net/url" + "strings" "go.sia.tech/core/types" "go.sia.tech/renterd/hostdb" @@ -23,6 +24,10 @@ var ( // ErrHostNotFound is returned when a host can't be retrieved from the // database. ErrHostNotFound = errors.New("host doesn't exist in hostdb") + + // ErrHostInfoNotFound is returned when host info can't be retrieved from + // the database. + ErrHostInfoNotFound = errors.New("host info doesn't exist in hostdb") ) type ( @@ -42,6 +47,7 @@ type ( MinRecentScanFailures uint64 `json:"minRecentScanFailures"` } + // SearchHostsRequest is the request type for the /hosts endpoint. SearchHostsRequest struct { Offset int `json:"offset"` Limit int `json:"limit"` @@ -50,6 +56,14 @@ type ( AddressContains string `json:"addressContains"` KeyIn []types.PublicKey `json:"keyIn"` } + + // UpdateHostInfoRequest is the request type for the PUT + // /autopilot/:id/host/:hostkey endpoint. + UpdateHostInfoRequest struct { + Gouging HostGougingBreakdown `json:"gouging"` + Score HostScoreBreakdown `json:"score"` + Usability HostUsabilityBreakdown `json:"usability"` + } ) type ( @@ -88,13 +102,6 @@ type ( } ) -func DefaultSearchHostOptions() SearchHostOptions { - return SearchHostOptions{ - Limit: -1, - FilterMode: HostFilterModeAll, - } -} - func (opts GetHostsOptions) Apply(values url.Values) { if opts.Offset != 0 { values.Set("offset", fmt.Sprint(opts.Offset)) @@ -115,3 +122,88 @@ func (opts HostsForScanningOptions) Apply(values url.Values) { values.Set("lastScan", fmt.Sprint(TimeRFC3339(opts.MaxLastScan))) } } + +type ( + HostInfo struct { + Host hostdb.Host `json:"host"` + Gouging HostGougingBreakdown `json:"gouging"` + Score HostScoreBreakdown `json:"score"` + Usability HostUsabilityBreakdown `json:"usability"` + } + + HostGougingBreakdown struct { + ContractErr string `json:"contractErr"` + DownloadErr string `json:"downloadErr"` + GougingErr string `json:"gougingErr"` + PruneErr string `json:"pruneErr"` + UploadErr string `json:"uploadErr"` + } + + HostScoreBreakdown struct { + Age float64 `json:"age"` + Collateral float64 `json:"collateral"` + Interactions float64 `json:"interactions"` + StorageRemaining float64 `json:"storageRemaining"` + Uptime float64 `json:"uptime"` + Version float64 `json:"version"` + Prices float64 `json:"prices"` + } + + HostUsabilityBreakdown struct { + Blocked bool `json:"blocked"` + Offline bool `json:"offline"` + LowScore bool `json:"lowScore"` + RedundantIP bool `json:"redundantIP"` + Gouging bool `json:"gouging"` + NotAcceptingContracts bool `json:"notAcceptingContracts"` + NotAnnounced bool `json:"notAnnounced"` + NotCompletingScan bool `json:"notCompletingScan"` + Unknown bool `json:"unknown"` + } +) + +func (sb HostScoreBreakdown) String() string { + return fmt.Sprintf("Age: %v, Col: %v, Int: %v, SR: %v, UT: %v, V: %v, Pr: %v", sb.Age, sb.Collateral, sb.Interactions, sb.StorageRemaining, sb.Uptime, sb.Version, sb.Prices) +} + +func (hgb HostGougingBreakdown) Gouging() bool { + for _, err := range []string{ + hgb.ContractErr, + hgb.DownloadErr, + hgb.GougingErr, + hgb.PruneErr, + hgb.UploadErr, + } { + if err != "" { + return true + } + } + return false +} + +func (hgb HostGougingBreakdown) String() string { + var reasons []string + for _, errStr := range []string{ + hgb.ContractErr, + hgb.DownloadErr, + hgb.GougingErr, + hgb.PruneErr, + hgb.UploadErr, + } { + if errStr != "" { + reasons = append(reasons, errStr) + } + } + return strings.Join(reasons, ";") +} + +func (sb HostScoreBreakdown) Score() float64 { + return sb.Age * sb.Collateral * sb.Interactions * sb.StorageRemaining * sb.Uptime * sb.Version * sb.Prices +} + +func (c AutopilotConfig) Validate() error { + if c.Hosts.MaxDowntimeHours > 99*365*24 { + return ErrMaxDowntimeHoursTooHigh + } + return nil +} diff --git a/bus/bus.go b/bus/bus.go index 05770eb96..c8875a092 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -104,6 +104,10 @@ type ( HostBlocklist(ctx context.Context) ([]string, error) UpdateHostAllowlistEntries(ctx context.Context, add, remove []types.PublicKey, clear bool) error UpdateHostBlocklistEntries(ctx context.Context, add, remove []string, clear bool) error + + HostInfo(ctx context.Context, autopilotID string, hk types.PublicKey) (api.HostInfo, error) + HostInfos(ctx context.Context, autopilotID string) ([]api.HostInfo, error) + UpdateHostInfo(ctx context.Context, autopilotID string, hk types.PublicKey, gouging api.HostGougingBreakdown, score api.HostScoreBreakdown, usability api.HostUsabilityBreakdown) error } // A MetadataStore stores information about contracts and objects. @@ -254,6 +258,10 @@ func (b *bus) Handler() http.Handler { "GET /autopilot/:id": b.autopilotsHandlerGET, "PUT /autopilot/:id": b.autopilotsHandlerPUT, + "GET /autopilot/:id/hosts": b.autopilotHostInfosHandlerGET, + "GET /autopilot/:id/host/:hostkey": b.autopilotHostInfoHandlerGET, + "PUT /autopilot/:id/host/:hostkey": b.autopilotHostInfoHandlerPUT, + "GET /buckets": b.bucketsHandlerGET, "POST /buckets": b.bucketsHandlerPOST, "PUT /bucket/:name/policy": b.bucketsHandlerPolicyPUT, @@ -1961,6 +1969,65 @@ func (b *bus) autopilotsHandlerPUT(jc jape.Context) { jc.Check("failed to update autopilot", b.as.UpdateAutopilot(jc.Request.Context(), ap)) } +func (b *bus) autopilotHostInfoHandlerGET(jc jape.Context) { + var id string + if jc.DecodeParam("id", &id) != nil { + return + } + var hostKey types.PublicKey + if jc.DecodeParam("hostkey", &hostKey) != nil { + return + } + + hi, err := b.hdb.HostInfo(jc.Request.Context(), id, hostKey) + if errors.Is(err, api.ErrAutopilotNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("failed to fetch host info", err) != nil { + return + } + jc.Encode(hi) +} + +func (b *bus) autopilotHostInfosHandlerGET(jc jape.Context) { + var id string + if jc.DecodeParam("id", &id) != nil { + return + } + + his, err := b.hdb.HostInfos(jc.Request.Context(), id) + if errors.Is(err, api.ErrAutopilotNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("failed to fetch host infos", err) != nil { + return + } + jc.Encode(his) +} + +func (b *bus) autopilotHostInfoHandlerPUT(jc jape.Context) { + var id string + if jc.DecodeParam("id", &id) != nil { + return + } + var hostKey types.PublicKey + if jc.DecodeParam("hostkey", &hostKey) != nil { + return + } + var hir api.UpdateHostInfoRequest + if jc.Check("failed to decode host info", jc.Decode(&hir)) != nil { + return + } + + err := b.hdb.UpdateHostInfo(jc.Request.Context(), id, hostKey, hir.Gouging, hir.Score, hir.Usability) + if errors.Is(err, api.ErrAutopilotNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("failed to update host info", err) != nil { + return + } +} + func (b *bus) contractTaxHandlerGET(jc jape.Context) { var payout types.Currency if jc.DecodeParam("payout", (*api.ParamCurrency)(&payout)) != nil { diff --git a/stores/hostdb.go b/stores/hostdb.go index 37aa18ee8..0a293f81f 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -80,6 +80,45 @@ type ( Blocklist []dbBlocklistEntry `gorm:"many2many:host_blocklist_entry_hosts;constraint:OnDelete:CASCADE"` } + // dbHostInfo contains information about a host that is collected and used + // by the autopilot. + dbHostInfo struct { + Model + + DBAutopilotID uint `gorm:"index:idx_host_infos_id,unique"` + DBAutopilot dbAutopilot + + DBHostID uint `gorm:"index:idx_host_infos_id,unique"` + DBHost dbHost + + // usability + UsabilityBlocked bool `gorm:"index:idx_host_infos_usability_blocked"` + UsabilityOffline bool `gorm:"index:idx_host_infos_usability_offline"` + UsabilityLowScore bool `gorm:"index:idx_host_infos_usability_low_score"` + UsabilityRedundantIP bool `gorm:"index:idx_host_infos_usability_redundant_ip"` + UsabilityGouging bool `gorm:"index:idx_host_infos_usability_gouging"` + UsabilityNotAcceptingContracts bool `gorm:"index:idx_host_infos_usability_not_accepting_contracts"` + UsabilityNotAnnounced bool `gorm:"index:idx_host_infos_usability_not_announced"` + UsabilityNotCompletingScan bool `gorm:"index:idx_host_infos_usability_not_completing_scan"` + UsabilityUnknown bool `gorm:"index:idx_host_infos_usability_unknown"` + + // score + ScoreAge float64 `gorm:"index:idx_host_infos_score_age"` + ScoreCollateral float64 `gorm:"index:idx_host_infos_score_collateral"` + ScoreInteractions float64 `gorm:"index:idx_host_infos_score_interactions"` + ScoreStorageRemaining float64 `gorm:"index:idx_host_infos_score_storage_remaining"` + ScoreUptime float64 `gorm:"index:idx_host_infos_score_uptime"` + ScoreVersion float64 `gorm:"index:idx_host_infos_score_version"` + ScorePrices float64 `gorm:"index:idx_host_infos_score_prices"` + + // gouging + GougingContractErr string + GougingDownloadErr string + GougingGougingErr string + GougingPruneErr string + GougingUploadErr string + } + // dbAllowlistEntry defines a table that stores the host blocklist. dbAllowlistEntry struct { Model @@ -275,6 +314,9 @@ func (dbConsensusInfo) TableName() string { return "consensus_infos" } // TableName implements the gorm.Tabler interface. func (dbHost) TableName() string { return "hosts" } +// TableName implements the gorm.Tabler interface. +func (dbHostInfo) TableName() string { return "host_infos" } + // TableName implements the gorm.Tabler interface. func (dbAllowlistEntry) TableName() string { return "host_allowlist_entries" } @@ -318,6 +360,70 @@ func (h dbHost) convert() hostdb.Host { } } +func (hi dbHostInfo) convert() api.HostInfo { + return api.HostInfo{ + Host: hi.DBHost.convert(), + Gouging: api.HostGougingBreakdown{ + ContractErr: hi.GougingContractErr, + DownloadErr: hi.GougingDownloadErr, + GougingErr: hi.GougingGougingErr, + PruneErr: hi.GougingPruneErr, + UploadErr: hi.GougingUploadErr, + }, + Score: api.HostScoreBreakdown{ + Age: hi.ScoreAge, + Collateral: hi.ScoreCollateral, + Interactions: hi.ScoreInteractions, + StorageRemaining: hi.ScoreStorageRemaining, + Uptime: hi.ScoreUptime, + Version: hi.ScoreVersion, + Prices: hi.ScorePrices, + }, + Usability: api.HostUsabilityBreakdown{ + Blocked: hi.UsabilityBlocked, + Offline: hi.UsabilityOffline, + LowScore: hi.UsabilityLowScore, + RedundantIP: hi.UsabilityRedundantIP, + Gouging: hi.UsabilityGouging, + NotAcceptingContracts: hi.UsabilityNotAcceptingContracts, + NotAnnounced: hi.UsabilityNotAnnounced, + NotCompletingScan: hi.UsabilityNotCompletingScan, + Unknown: hi.UsabilityUnknown, + }, + } +} + +func convertHostInfo(apID, hID uint, gouging api.HostGougingBreakdown, score api.HostScoreBreakdown, usability api.HostUsabilityBreakdown) *dbHostInfo { + return &dbHostInfo{ + DBAutopilotID: apID, + DBHostID: hID, + + UsabilityBlocked: usability.Blocked, + UsabilityOffline: usability.Offline, + UsabilityLowScore: usability.LowScore, + UsabilityRedundantIP: usability.RedundantIP, + UsabilityGouging: usability.Gouging, + UsabilityNotAcceptingContracts: usability.NotAcceptingContracts, + UsabilityNotAnnounced: usability.NotAnnounced, + UsabilityNotCompletingScan: usability.NotCompletingScan, + UsabilityUnknown: usability.Unknown, + + ScoreAge: score.Age, + ScoreCollateral: score.Collateral, + ScoreInteractions: score.Interactions, + ScoreStorageRemaining: score.StorageRemaining, + ScoreUptime: score.Uptime, + ScoreVersion: score.Version, + ScorePrices: score.Prices, + + GougingContractErr: gouging.ContractErr, + GougingDownloadErr: gouging.DownloadErr, + GougingGougingErr: gouging.GougingErr, + GougingPruneErr: gouging.PruneErr, + GougingUploadErr: gouging.UploadErr, + } +} + func (h *dbHost) BeforeCreate(tx *gorm.DB) (err error) { tx.Statement.AddClause(clause.OnConflict{ Columns: []clause.Column{{Name: "public_key"}}, @@ -443,6 +549,126 @@ func (ss *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (hostdb.H }, nil } +func (ss *SQLStore) HostInfo(ctx context.Context, autopilotID string, hk types.PublicKey) (hi api.HostInfo, err error) { + err = ss.db.Transaction(func(tx *gorm.DB) error { + // fetch ap id + var apID uint + if err := tx. + Model(&dbAutopilot{}). + Where("identifier = ?", autopilotID). + Select("id"). + Take(&apID). + Error; errors.Is(err, gorm.ErrRecordNotFound) { + return api.ErrAutopilotNotFound + } else if err != nil { + return err + } + + // fetch host id + var hID uint + if err := tx. + Model(&dbHost{}). + Where("public_key = ?", publicKey(hk)). + Select("id"). + Take(&hID). + Error; errors.Is(err, gorm.ErrRecordNotFound) { + return api.ErrHostNotFound + } else if err != nil { + return err + } + + // fetch host info + var entity dbHostInfo + if err := tx. + Model(&dbHostInfo{}). + Where("db_autopilot_id = ? AND db_host_id = ?", apID, hID). + Preload("DBHost"). + First(&entity). + Error; errors.Is(err, gorm.ErrRecordNotFound) { + return api.ErrHostInfoNotFound + } else if err != nil { + return err + } + + hi = entity.convert() + return nil + }) + return +} + +func (ss *SQLStore) HostInfos(ctx context.Context, autopilotID string) (his []api.HostInfo, err error) { + err = ss.db.Transaction(func(tx *gorm.DB) error { + // fetch ap id + var apID uint + if err := tx. + Model(&dbAutopilot{}). + Where("identifier = ?", autopilotID). + Select("id"). + Take(&apID). + Error; errors.Is(err, gorm.ErrRecordNotFound) { + return api.ErrAutopilotNotFound + } else if err != nil { + return err + } + + // fetch host info + var infos []dbHostInfo + if err := tx. + Model(&dbHostInfo{}). + Where("db_autopilot_id = ?", apID). + Preload("DBHost"). + Find(&infos). + Error; err != nil { + return err + } + for _, hi := range infos { + his = append(his, hi.convert()) + } + return nil + }) + return +} + +func (ss *SQLStore) UpdateHostInfo(ctx context.Context, autopilotID string, hk types.PublicKey, gouging api.HostGougingBreakdown, score api.HostScoreBreakdown, usability api.HostUsabilityBreakdown) (err error) { + err = ss.db.Transaction(func(tx *gorm.DB) error { + // fetch ap id + var apID uint + if err := tx. + Model(&dbAutopilot{}). + Where("identifier = ?", autopilotID). + Select("id"). + Take(&apID). + Error; errors.Is(err, gorm.ErrRecordNotFound) { + return api.ErrAutopilotNotFound + } else if err != nil { + return err + } + + // fetch host id + var hID uint + if err := tx. + Model(&dbHost{}). + Where("public_key = ?", publicKey(hk)). + Select("id"). + Take(&hID). + Error; errors.Is(err, gorm.ErrRecordNotFound) { + return api.ErrHostNotFound + } else if err != nil { + return err + } + + // update host info + return tx. + Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "db_autopilot_id"}, {Name: "db_host_id"}}, + UpdateAll: true, + }). + Create(convertHostInfo(apID, hID, gouging, score, usability)). + Error + }) + return +} + // HostsForScanning returns the address of hosts for scanning. func (ss *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]hostdb.HostAddress, error) { if offset < 0 { diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index 35872ea2d..06692ad8c 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -1064,6 +1064,95 @@ func TestAnnouncementMaxAge(t *testing.T) { } } +func TestHostInfo(t *testing.T) { + ss := newTestSQLStore(t, defaultTestSQLStoreConfig) + defer ss.Close() + + // fetch info for a non-existing autopilot + _, err := ss.HostInfo(context.Background(), "foo", types.PublicKey{1}) + if !errors.Is(err, api.ErrAutopilotNotFound) { + t.Fatal(err) + } + + // add autopilot + err = ss.UpdateAutopilot(context.Background(), api.Autopilot{ID: "foo"}) + if err != nil { + t.Fatal(err) + } + + // fetch info for a non-existing host + _, err = ss.HostInfo(context.Background(), "foo", types.PublicKey{1}) + if !errors.Is(err, api.ErrHostNotFound) { + t.Fatal(err) + } + + // add host + err = ss.addTestHost(types.PublicKey{1}) + if err != nil { + t.Fatal(err) + } + h, err := ss.Host(context.Background(), types.PublicKey{1}) + if err != nil { + t.Fatal(err) + } + + // fetch non-existing info + _, err = ss.HostInfo(context.Background(), "foo", types.PublicKey{1}) + if !errors.Is(err, api.ErrHostInfoNotFound) { + t.Fatal(err) + } + + // add host info + want := newTestHostInfo(h.Host) + err = ss.UpdateHostInfo(context.Background(), "foo", types.PublicKey{1}, want.Gouging, want.Score, want.Usability) + if err != nil { + t.Fatal(err) + } + + // fetch info + got, err := ss.HostInfo(context.Background(), "foo", types.PublicKey{1}) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(got, want) { + t.Fatal("mismatch", cmp.Diff(got, want)) + } + + // update info + want.Score.Age = 0 + err = ss.UpdateHostInfo(context.Background(), "foo", types.PublicKey{1}, want.Gouging, want.Score, want.Usability) + if err != nil { + t.Fatal(err) + } + + // fetch info + got, err = ss.HostInfo(context.Background(), "foo", types.PublicKey{1}) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(got, want) { + t.Fatal("mismatch") + } + + // add another host info + err = ss.addTestHost(types.PublicKey{2}) + if err != nil { + t.Fatal(err) + } + err = ss.UpdateHostInfo(context.Background(), "foo", types.PublicKey{2}, want.Gouging, want.Score, want.Usability) + if err != nil { + t.Fatal(err) + } + + // fetch all infos for autopilot + his, err := ss.HostInfos(context.Background(), "foo") + if err != nil { + t.Fatal(err) + } else if len(his) != 2 { + t.Fatal("unexpected") + } else if his[0].Host.PublicKey != (types.PublicKey{1}) || his[1].Host.PublicKey != (types.PublicKey{2}) { + t.Fatal("unexpected", his) + } +} + // addTestHosts adds 'n' hosts to the db and returns their keys. func (s *SQLStore) addTestHosts(n int) (keys []types.PublicKey, err error) { cnt, err := s.contractsCount() @@ -1156,3 +1245,36 @@ func newTestTransaction(ha modules.HostAnnouncement, sk types.PrivateKey) stypes buf.Write(encoding.Marshal(sk.SignHash(types.Hash256(crypto.HashObject(ha))))) return stypes.Transaction{ArbitraryData: [][]byte{buf.Bytes()}} } + +func newTestHostInfo(h hostdb.Host) api.HostInfo { + return api.HostInfo{ + Host: h, + Gouging: api.HostGougingBreakdown{ + ContractErr: "foo", + DownloadErr: "bar", + GougingErr: "baz", + PruneErr: "qux", + UploadErr: "quuz", + }, + Score: api.HostScoreBreakdown{ + Age: .1, + Collateral: .2, + Interactions: .3, + StorageRemaining: .4, + Uptime: .5, + Version: .6, + Prices: .7, + }, + Usability: api.HostUsabilityBreakdown{ + Blocked: true, + Offline: true, + LowScore: true, + RedundantIP: true, + Gouging: true, + NotAcceptingContracts: true, + NotAnnounced: true, + NotCompletingScan: true, + Unknown: true, + }, + } +} diff --git a/stores/migrations.go b/stores/migrations.go index b0304090e..e1d298ab8 100644 --- a/stores/migrations.go +++ b/stores/migrations.go @@ -62,6 +62,12 @@ func performMigrations(db *gorm.DB, logger *zap.SugaredLogger) error { return performMigration(tx, dbIdentifier, "00006_idx_objects_created_at", logger) }, }, + { + ID: "00007_host_info", + Migrate: func(tx *gorm.DB) error { + return performMigration(tx, dbIdentifier, "00007_host_info", logger) + }, + }, } // Create migrator. diff --git a/stores/migrations/mysql/main/migration_00007_host_info.sql b/stores/migrations/mysql/main/migration_00007_host_info.sql new file mode 100644 index 000000000..69864b3e1 --- /dev/null +++ b/stores/migrations/mysql/main/migration_00007_host_info.sql @@ -0,0 +1,54 @@ +-- dbHostInfo +CREATE TABLE `host_infos` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `created_at` datetime(3) DEFAULT NULL, + + `db_autopilot_id` bigint unsigned NOT NULL, + `db_host_id` bigint unsigned NOT NULL, + + `usability_blocked` boolean NOT NULL DEFAULT false, + `usability_offline` boolean NOT NULL DEFAULT false, + `usability_low_score` boolean NOT NULL DEFAULT false, + `usability_redundant_ip` boolean NOT NULL DEFAULT false, + `usability_gouging` boolean NOT NULL DEFAULT false, + `usability_not_accepting_contracts` boolean NOT NULL DEFAULT false, + `usability_not_announced` boolean NOT NULL DEFAULT false, + `usability_not_completing_scan` boolean NOT NULL DEFAULT false, + `usability_unknown` boolean NOT NULL DEFAULT false, + + `score_age` double NOT NULL, + `score_collateral` double NOT NULL, + `score_interactions` double NOT NULL, + `score_storage_remaining` double NOT NULL, + `score_uptime` double NOT NULL, + `score_version` double NOT NULL, + `score_prices` double NOT NULL, + + `gouging_contract_err` text, + `gouging_download_err` text, + `gouging_gouging_err` text, + `gouging_prune_err` text, + `gouging_upload_err` text, + + PRIMARY KEY (`id`), + UNIQUE KEY `idx_host_infos_id` (`db_autopilot_id`, `db_host_id`), + INDEX `idx_host_infos_usability_blocked` (`usability_blocked`), + INDEX `idx_host_infos_usability_offline` (`usability_offline`), + INDEX `idx_host_infos_usability_low_score` (`usability_low_score`), + INDEX `idx_host_infos_usability_redundant_ip` (`usability_redundant_ip`), + INDEX `idx_host_infos_usability_gouging` (`usability_gouging`), + INDEX `idx_host_infos_usability_not_accepting_contracts` (`usability_not_accepting_contracts`), + INDEX `idx_host_infos_usability_not_announced` (`usability_not_announced`), + INDEX `idx_host_infos_usability_not_completing_scan` (`usability_not_completing_scan`), + INDEX `idx_host_infos_usability_unknown` (`usability_unknown`), + INDEX `idx_host_infos_score_age` (`score_age`), + INDEX `idx_host_infos_score_collateral` (`score_collateral`), + INDEX `idx_host_infos_score_interactions` (`score_interactions`), + INDEX `idx_host_infos_score_storage_remaining` (`score_storage_remaining`), + INDEX `idx_host_infos_score_uptime` (`score_uptime`), + INDEX `idx_host_infos_score_version` (`score_version`), + INDEX `idx_host_infos_score_prices` (`score_prices`), + + CONSTRAINT `fk_host_infos_autopilot` FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_host_infos_host` FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; diff --git a/stores/migrations/mysql/main/schema.sql b/stores/migrations/mysql/main/schema.sql index 68b42ae47..4eaa91499 100644 --- a/stores/migrations/mysql/main/schema.sql +++ b/stores/migrations/mysql/main/schema.sql @@ -422,5 +422,60 @@ CREATE TABLE `object_user_metadata` ( CONSTRAINT `fk_multipart_upload_user_metadata` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads` (`id`) ON DELETE SET NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +-- dbHostInfo +CREATE TABLE `host_infos` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `created_at` datetime(3) DEFAULT NULL, + + `db_autopilot_id` bigint unsigned NOT NULL, + `db_host_id` bigint unsigned NOT NULL, + + `usability_blocked` boolean NOT NULL DEFAULT false, + `usability_offline` boolean NOT NULL DEFAULT false, + `usability_low_score` boolean NOT NULL DEFAULT false, + `usability_redundant_ip` boolean NOT NULL DEFAULT false, + `usability_gouging` boolean NOT NULL DEFAULT false, + `usability_not_accepting_contracts` boolean NOT NULL DEFAULT false, + `usability_not_announced` boolean NOT NULL DEFAULT false, + `usability_not_completing_scan` boolean NOT NULL DEFAULT false, + `usability_unknown` boolean NOT NULL DEFAULT false, + + `score_age` double NOT NULL, + `score_collateral` double NOT NULL, + `score_interactions` double NOT NULL, + `score_storage_remaining` double NOT NULL, + `score_uptime` double NOT NULL, + `score_version` double NOT NULL, + `score_prices` double NOT NULL, + + `gouging_contract_err` text, + `gouging_download_err` text, + `gouging_gouging_err` text, + `gouging_prune_err` text, + `gouging_upload_err` text, + + PRIMARY KEY (`id`), + UNIQUE KEY `idx_host_infos_id` (`db_autopilot_id`, `db_host_id`), + INDEX `idx_host_infos_usability_blocked` (`usability_blocked`), + INDEX `idx_host_infos_usability_offline` (`usability_offline`), + INDEX `idx_host_infos_usability_low_score` (`usability_low_score`), + INDEX `idx_host_infos_usability_redundant_ip` (`usability_redundant_ip`), + INDEX `idx_host_infos_usability_gouging` (`usability_gouging`), + INDEX `idx_host_infos_usability_not_accepting_contracts` (`usability_not_accepting_contracts`), + INDEX `idx_host_infos_usability_not_announced` (`usability_not_announced`), + INDEX `idx_host_infos_usability_not_completing_scan` (`usability_not_completing_scan`), + INDEX `idx_host_infos_usability_unknown` (`usability_unknown`), + INDEX `idx_host_infos_score_age` (`score_age`), + INDEX `idx_host_infos_score_collateral` (`score_collateral`), + INDEX `idx_host_infos_score_interactions` (`score_interactions`), + INDEX `idx_host_infos_score_storage_remaining` (`score_storage_remaining`), + INDEX `idx_host_infos_score_uptime` (`score_uptime`), + INDEX `idx_host_infos_score_version` (`score_version`), + INDEX `idx_host_infos_score_prices` (`score_prices`), + + CONSTRAINT `fk_host_infos_autopilot` FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_host_infos_host` FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + -- create default bucket INSERT INTO buckets (created_at, name) VALUES (CURRENT_TIMESTAMP, 'default'); \ No newline at end of file diff --git a/stores/migrations/sqlite/main/migration_00007_host_info.sql b/stores/migrations/sqlite/main/migration_00007_host_info.sql new file mode 100644 index 000000000..5d425dfe7 --- /dev/null +++ b/stores/migrations/sqlite/main/migration_00007_host_info.sql @@ -0,0 +1,54 @@ +-- dbHostInfo +CREATE TABLE `host_infos` ( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `created_at` datetime, + + `db_autopilot_id` INTEGER NOT NULL, + `db_host_id` INTEGER NOT NULL, + + `usability_blocked` INTEGER NOT NULL DEFAULT 0, + `usability_offline` INTEGER NOT NULL DEFAULT 0, + `usability_low_score` INTEGER NOT NULL DEFAULT 0, + `usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, + `usability_gouging` INTEGER NOT NULL DEFAULT 0, + `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, + `usability_not_announced` INTEGER NOT NULL DEFAULT 0, + `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, + `usability_unknown` INTEGER NOT NULL DEFAULT 0, + + `score_age` REAL NOT NULL, + `score_collateral` REAL NOT NULL, + `score_interactions` REAL NOT NULL, + `score_storage_remaining` REAL NOT NULL, + `score_uptime` REAL NOT NULL, + `score_version` REAL NOT NULL, + `score_prices` REAL NOT NULL, + + `gouging_contract_err` TEXT, + `gouging_download_err` TEXT, + `gouging_gouging_err` TEXT, + `gouging_prune_err` TEXT, + `gouging_upload_err` TEXT, + + FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, + FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE +); + +-- Indexes creation +CREATE UNIQUE INDEX `idx_host_infos_id` ON `host_infos` (`db_autopilot_id`, `db_host_id`); +CREATE INDEX `idx_host_infos_usability_blocked` ON `host_infos` (`usability_blocked`); +CREATE INDEX `idx_host_infos_usability_offline` ON `host_infos` (`usability_offline`); +CREATE INDEX `idx_host_infos_usability_low_score` ON `host_infos` (`usability_low_score`); +CREATE INDEX `idx_host_infos_usability_redundant_ip` ON `host_infos` (`usability_redundant_ip`); +CREATE INDEX `idx_host_infos_usability_gouging` ON `host_infos` (`usability_gouging`); +CREATE INDEX `idx_host_infos_usability_not_accepting_contracts` ON `host_infos` (`usability_not_accepting_contracts`); +CREATE INDEX `idx_host_infos_usability_not_announced` ON `host_infos` (`usability_not_announced`); +CREATE INDEX `idx_host_infos_usability_not_completing_scan` ON `host_infos` (`usability_not_completing_scan`); +CREATE INDEX `idx_host_infos_usability_unknown` ON `host_infos` (`usability_unknown`); +CREATE INDEX `idx_host_infos_score_age` ON `host_infos` (`score_age`); +CREATE INDEX `idx_host_infos_score_collateral` ON `host_infos` (`score_collateral`); +CREATE INDEX `idx_host_infos_score_interactions` ON `host_infos` (`score_interactions`); +CREATE INDEX `idx_host_infos_score_storage_remaining` ON `host_infos` (`score_storage_remaining`); +CREATE INDEX `idx_host_infos_score_uptime` ON `host_infos` (`score_uptime`); +CREATE INDEX `idx_host_infos_score_version` ON `host_infos` (`score_version`); +CREATE INDEX `idx_host_infos_score_prices` ON `host_infos` (`score_prices`); diff --git a/stores/migrations/sqlite/main/schema.sql b/stores/migrations/sqlite/main/schema.sql index 9875e81e3..5ec7a2b0e 100644 --- a/stores/migrations/sqlite/main/schema.sql +++ b/stores/migrations/sqlite/main/schema.sql @@ -149,5 +149,25 @@ CREATE UNIQUE INDEX `idx_module_event_url` ON `webhooks`(`module`,`event`,`url`) CREATE TABLE `object_user_metadata` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`db_object_id` integer DEFAULT NULL,`db_multipart_upload_id` integer DEFAULT NULL,`key` text NOT NULL,`value` text, CONSTRAINT `fk_object_user_metadata` FOREIGN KEY (`db_object_id`) REFERENCES `objects` (`id`) ON DELETE CASCADE, CONSTRAINT `fk_multipart_upload_user_metadata` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads` (`id`) ON DELETE SET NULL); CREATE UNIQUE INDEX `idx_object_user_metadata_key` ON `object_user_metadata`(`db_object_id`,`db_multipart_upload_id`,`key`); +-- dbHostInfo +CREATE TABLE `host_infos` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `created_at` datetime, `db_autopilot_id` INTEGER NOT NULL, `db_host_id` INTEGER NOT NULL, `usability_blocked` INTEGER NOT NULL DEFAULT 0, `usability_offline` INTEGER NOT NULL DEFAULT 0, `usability_low_score` INTEGER NOT NULL DEFAULT 0, `usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, `usability_gouging` INTEGER NOT NULL DEFAULT 0, `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, `usability_not_announced` INTEGER NOT NULL DEFAULT 0, `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, `usability_unknown` INTEGER NOT NULL DEFAULT 0, `score_age` REAL NOT NULL, `score_collateral` REAL NOT NULL, `score_interactions` REAL NOT NULL, `score_storage_remaining` REAL NOT NULL, `score_uptime` REAL NOT NULL, `score_version` REAL NOT NULL, `score_prices` REAL NOT NULL, `gouging_contract_err` TEXT, `gouging_download_err` TEXT, `gouging_gouging_err` TEXT, `gouging_prune_err` TEXT, `gouging_upload_err` TEXT, FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE); +CREATE UNIQUE INDEX `idx_host_infos_id` ON `host_infos` (`db_autopilot_id`, `db_host_id`); +CREATE INDEX `idx_host_infos_usability_blocked` ON `host_infos` (`usability_blocked`); +CREATE INDEX `idx_host_infos_usability_offline` ON `host_infos` (`usability_offline`); +CREATE INDEX `idx_host_infos_usability_low_score` ON `host_infos` (`usability_low_score`); +CREATE INDEX `idx_host_infos_usability_redundant_ip` ON `host_infos` (`usability_redundant_ip`); +CREATE INDEX `idx_host_infos_usability_gouging` ON `host_infos` (`usability_gouging`); +CREATE INDEX `idx_host_infos_usability_not_accepting_contracts` ON `host_infos` (`usability_not_accepting_contracts`); +CREATE INDEX `idx_host_infos_usability_not_announced` ON `host_infos` (`usability_not_announced`); +CREATE INDEX `idx_host_infos_usability_not_completing_scan` ON `host_infos` (`usability_not_completing_scan`); +CREATE INDEX `idx_host_infos_usability_unknown` ON `host_infos` (`usability_unknown`); +CREATE INDEX `idx_host_infos_score_age` ON `host_infos` (`score_age`); +CREATE INDEX `idx_host_infos_score_collateral` ON `host_infos` (`score_collateral`); +CREATE INDEX `idx_host_infos_score_interactions` ON `host_infos` (`score_interactions`); +CREATE INDEX `idx_host_infos_score_storage_remaining` ON `host_infos` (`score_storage_remaining`); +CREATE INDEX `idx_host_infos_score_uptime` ON `host_infos` (`score_uptime`); +CREATE INDEX `idx_host_infos_score_version` ON `host_infos` (`score_version`); +CREATE INDEX `idx_host_infos_score_prices` ON `host_infos` (`score_prices`); + -- create default bucket INSERT INTO buckets (created_at, name) VALUES (CURRENT_TIMESTAMP, 'default'); From d70d47faefb45af8f635c9b44e1dea523f30b052 Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 18 Mar 2024 21:28:13 +0100 Subject: [PATCH 02/23] autopilot: calculate host info and update in the bus --- api/host.go | 65 +++++++++++++ autopilot/autopilot.go | 61 ++++++++---- autopilot/contractor.go | 101 +++++++++++--------- autopilot/hostfilter.go | 156 +++++++++--------------------- autopilot/hostinfo.go | 206 ---------------------------------------- bus/bus.go | 52 +++++++--- bus/client/hosts.go | 29 ++++++ stores/hostdb.go | 117 +++++++++++++++++------ stores/hostdb_test.go | 107 ++++++++++++++++++++- worker/upload.go | 1 - 10 files changed, 468 insertions(+), 427 deletions(-) delete mode 100644 autopilot/hostinfo.go diff --git a/api/host.go b/api/host.go index 05905d5b3..18ae2cc38 100644 --- a/api/host.go +++ b/api/host.go @@ -30,6 +30,19 @@ var ( ErrHostInfoNotFound = errors.New("host info doesn't exist in hostdb") ) +var ( + ErrUsabilityHostBlocked = errors.New("host is blocked") + ErrUsabilityHostNotFound = errors.New("host not found") + ErrUsabilityHostOffline = errors.New("host is offline") + ErrUsabilityHostLowScore = errors.New("host's score is below minimum") + ErrUsabilityHostRedundantIP = errors.New("host has redundant IP") + ErrUsabilityHostPriceGouging = errors.New("host is price gouging") + ErrUsabilityHostNotAcceptingContracts = errors.New("host is not accepting contracts") + ErrUsabilityHostNotCompletingScan = errors.New("host is not completing scan") + ErrUsabilityHostNotAnnounced = errors.New("host is not announced") + ErrUsabilityUnknown = errors.New("unknown") +) + type ( // HostsScanRequest is the request type for the /hosts/scans endpoint. HostsScanRequest struct { @@ -57,6 +70,10 @@ type ( KeyIn []types.PublicKey `json:"keyIn"` } + // HostInfosRequest is the request type for the POST /autopilot/:id/hosts + // endpoint. + HostInfosRequest SearchHostsRequest + // UpdateHostInfoRequest is the request type for the PUT // /autopilot/:id/host/:hostkey endpoint. UpdateHostInfoRequest struct { @@ -100,6 +117,10 @@ type ( Limit int Offset int } + HostInfoOptions struct { + SearchHostOptions + UsabilityMode string + } ) func (opts GetHostsOptions) Apply(values url.Values) { @@ -201,6 +222,50 @@ func (sb HostScoreBreakdown) Score() float64 { return sb.Age * sb.Collateral * sb.Interactions * sb.StorageRemaining * sb.Uptime * sb.Version * sb.Prices } +func (ub HostUsabilityBreakdown) Usable() bool { + return !ub.Blocked && + !ub.Offline && + !ub.LowScore && + !ub.RedundantIP && + !ub.Gouging && + !ub.NotAcceptingContracts && + !ub.NotAnnounced && + !ub.NotCompletingScan && + !ub.Unknown +} + +func (ub HostUsabilityBreakdown) UnusableReasons() []string { + var reasons []string + if ub.Blocked { + reasons = append(reasons, ErrUsabilityHostBlocked.Error()) + } + if ub.Offline { + reasons = append(reasons, ErrUsabilityHostOffline.Error()) + } + if ub.LowScore { + reasons = append(reasons, ErrUsabilityHostLowScore.Error()) + } + if ub.RedundantIP { + reasons = append(reasons, ErrUsabilityHostRedundantIP.Error()) + } + if ub.Gouging { + reasons = append(reasons, ErrUsabilityHostPriceGouging.Error()) + } + if ub.NotAcceptingContracts { + reasons = append(reasons, ErrUsabilityHostNotAcceptingContracts.Error()) + } + if ub.NotAnnounced { + reasons = append(reasons, ErrUsabilityHostNotAnnounced.Error()) + } + if ub.NotCompletingScan { + reasons = append(reasons, ErrUsabilityHostNotCompletingScan.Error()) + } + if ub.Unknown { + reasons = append(reasons, ErrUsabilityUnknown.Error()) + } + return reasons +} + func (c AutopilotConfig) Validate() error { if c.Hosts.MaxDowntimeHours > 99*365*24 { return ErrMaxDowntimeHoursTooHigh diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index eb08c9456..2514c7a78 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -59,6 +59,10 @@ type Bus interface { RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.Host, error) + HostInfo(ctx context.Context, autopilotID string, hostKey types.PublicKey) (api.HostInfo, error) + HostInfos(ctx context.Context, autopilotID string, opts api.HostInfoOptions) ([]api.HostInfo, error) + UpdateHostInfo(ctx context.Context, autopilotID string, hostKey types.PublicKey, gouging api.HostGougingBreakdown, score api.HostScoreBreakdown, usability api.HostUsabilityBreakdown) error + // metrics RecordContractSetChurnMetric(ctx context.Context, metrics ...api.ContractSetChurnMetric) error RecordContractPruneMetric(ctx context.Context, metrics ...api.ContractPruneMetric) error @@ -685,7 +689,7 @@ func (ap *Autopilot) hostHandlerGET(jc jape.Context) { return } - host, err := ap.c.HostInfo(jc.Request.Context(), hostKey) + host, err := ap.bus.HostInfo(jc.Request.Context(), ap.id, hostKey) if jc.Check("failed to get host info", err) != nil { return } @@ -724,22 +728,45 @@ func (ap *Autopilot) stateHandlerGET(jc jape.Context) { } func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { - var req api.SearchHostsRequest + var req api.HostInfosRequest if jc.Decode(&req) != nil { return } - hosts, err := ap.c.HostInfos(jc.Request.Context(), req.FilterMode, req.UsabilityMode, req.AddressContains, req.KeyIn, req.Offset, req.Limit) + hosts, err := ap.bus.HostInfos(jc.Request.Context(), ap.id, api.HostInfoOptions{ + UsabilityMode: req.UsabilityMode, + SearchHostOptions: api.SearchHostOptions{ + FilterMode: req.FilterMode, + AddressContains: req.AddressContains, + KeyIn: req.KeyIn, + Offset: req.Offset, + Limit: req.Limit, + }, + }) if jc.Check("failed to get host info", err) != nil { return } - jc.Encode(hosts) + resps := make([]api.HostHandlerResponse, len(hosts)) + for i, host := range hosts { + resps[i] = api.HostHandlerResponse{ + Host: host.Host, + Checks: &api.HostHandlerResponseChecks{ + Gouging: host.Gouging.Gouging(), + GougingBreakdown: host.Gouging, + Score: host.Score.Score(), + ScoreBreakdown: host.Score, + Usable: host.Usability.Usable(), + UnusableReasons: host.Usability.UnusableReasons(), + }, + } + } + jc.Encode(resps) } func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.Host) (usables uint64) { gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow) for _, host := range hosts { - usable, _ := isUsableHost(cfg, rs, gc, host, smallestValidScore, 0) - if usable { + hi := calculateHostInfo(cfg, rs, gc, host, smallestValidScore, 0) + if hi.Usability.Usable() { usables++ } } @@ -754,36 +781,36 @@ func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Cu resp.Hosts = uint64(len(hosts)) for _, host := range hosts { - usable, usableBreakdown := isUsableHost(cfg, rs, gc, host, 0, 0) - if usable { + hi := calculateHostInfo(cfg, rs, gc, host, 0, 0) + if hi.Usability.Usable() { resp.Usable++ continue } - if usableBreakdown.blocked > 0 { + if hi.Usability.Blocked { resp.Unusable.Blocked++ } - if usableBreakdown.notacceptingcontracts > 0 { + if hi.Usability.NotAcceptingContracts { resp.Unusable.NotAcceptingContracts++ } - if usableBreakdown.notcompletingscan > 0 { + if hi.Usability.NotCompletingScan { resp.Unusable.NotScanned++ } - if usableBreakdown.unknown > 0 { + if hi.Usability.Unknown { resp.Unusable.Unknown++ } - if usableBreakdown.gougingBreakdown.ContractErr != "" { + if hi.Gouging.ContractErr != "" { resp.Unusable.Gouging.Contract++ } - if usableBreakdown.gougingBreakdown.DownloadErr != "" { + if hi.Gouging.DownloadErr != "" { resp.Unusable.Gouging.Download++ } - if usableBreakdown.gougingBreakdown.GougingErr != "" { + if hi.Gouging.GougingErr != "" { resp.Unusable.Gouging.Gouging++ } - if usableBreakdown.gougingBreakdown.PruneErr != "" { + if hi.Gouging.PruneErr != "" { resp.Unusable.Gouging.Pruning++ } - if usableBreakdown.gougingBreakdown.UploadErr != "" { + if hi.Gouging.UploadErr != "" { resp.Unusable.Gouging.Upload++ } } diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 8dfc702f3..810ca093c 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -101,16 +101,10 @@ type ( pruning bool pruningLastStart time.Time - cachedHostInfo map[types.PublicKey]hostInfo cachedDataStored map[types.PublicKey]uint64 cachedMinScore float64 } - hostInfo struct { - Usable bool - UnusableResult unusableHostResult - } - scoredHost struct { host hostdb.Host score float64 @@ -281,35 +275,19 @@ func (c *contractor) performContractMaintenance(ctx context.Context, w Worker) ( c.logger.Warn("could not calculate min score, no hosts found") } - // fetch consensus state - cs, err := c.ap.bus.ConsensusState(ctx) - if err != nil { - return false, err - } - - // create gouging checker - gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) - - // prepare hosts for cache - hostInfos := make(map[types.PublicKey]hostInfo) - for _, h := range hosts { - // ignore the pricetable's HostBlockHeight by setting it to our own blockheight - h.PriceTable.HostBlockHeight = cs.BlockHeight - isUsable, unusableResult := isUsableHost(state.cfg, state.rs, gc, h, minScore, hostData[h.PublicKey]) - hostInfos[h.PublicKey] = hostInfo{ - Usable: isUsable, - UnusableResult: unusableResult, - } - } - // update cache. c.mu.Lock() - c.cachedHostInfo = hostInfos c.cachedDataStored = hostData c.cachedMinScore = minScore c.mu.Unlock() - // run checks + // run host checks + err = c.runHostChecks(ctx, hosts) + if err != nil { + return false, fmt.Errorf("failed to run host checks, err: %v", err) + } + + // run contract checks updatedSet, toArchive, toStopUsing, toRefresh, toRenew, err := c.runContractChecks(ctx, w, contracts, isInCurrentSet, minScore) if err != nil { return false, fmt.Errorf("failed to run contract checks, err: %v", err) @@ -741,15 +719,15 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts host, err := c.ap.bus.Host(ctx, hk) if err != nil { c.logger.Errorw(fmt.Sprintf("missing host, err: %v", err), "hk", hk) - toStopUsing[fcid] = errHostNotFound.Error() + toStopUsing[fcid] = api.ErrUsabilityHostNotFound.Error() notfound++ continue } // if the host is blocked we ignore it, it might be unblocked later if host.Blocked { - c.logger.Infow("unusable host", "hk", hk, "fcid", fcid, "reasons", errHostBlocked.Error()) - toStopUsing[fcid] = errHostBlocked.Error() + c.logger.Infow("unusable host", "hk", hk, "fcid", fcid, "reasons", api.ErrUsabilityHostBlocked.Error()) + toStopUsing[fcid] = api.ErrUsabilityHostBlocked.Error() continue } @@ -780,9 +758,9 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts host.PriceTable.HostBlockHeight = cs.BlockHeight // decide whether the host is still good - usable, unusableResult := isUsableHost(state.cfg, state.rs, gc, host.Host, minScore, contract.FileSize()) - if !usable { - reasons := unusableResult.reasons() + hi := calculateHostInfo(state.cfg, state.rs, gc, host.Host, minScore, contract.FileSize()) + if !hi.Usability.Usable() { + reasons := hi.Usability.UnusableReasons() toStopUsing[fcid] = strings.Join(reasons, ",") c.logger.Infow("unusable host", "hk", hk, "fcid", fcid, "reasons", reasons) continue @@ -795,7 +773,7 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts if _, found := inCurrentSet[fcid]; !found || remainingKeepLeeway == 0 { toStopUsing[fcid] = errContractNoRevision.Error() } else if !state.cfg.Hosts.AllowRedundantIPs && ipFilter.IsRedundantIP(contract.HostIP, contract.HostKey) { - toStopUsing[fcid] = fmt.Sprintf("%v; %v", errHostRedundantIP, errContractNoRevision) + toStopUsing[fcid] = fmt.Sprintf("%v; %v", api.ErrUsabilityHostRedundantIP, errContractNoRevision) } else { toKeep = append(toKeep, contract.ContractMetadata) remainingKeepLeeway-- // we let it slide @@ -847,7 +825,38 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts return toKeep, toArchive, toStopUsing, toRefresh, toRenew, nil } -func (c *contractor) runContractFormations(ctx context.Context, w Worker, candidates scoredHosts, usedHosts map[types.PublicKey]struct{}, unusableHosts unusableHostResult, missing uint64, budget *types.Currency) (formed []api.ContractMetadata, _ error) { +func (c *contractor) runHostChecks(ctx context.Context, hosts []hostdb.Host) error { + // convenience variables + state := c.ap.State() + + // fetch consensus state + cs, err := c.ap.bus.ConsensusState(ctx) + if err != nil { + return err + } + + // create gouging checker + gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) + + // grab min score and host data from cache + c.mu.Lock() + minScore := c.cachedMinScore + hostData := c.cachedDataStored + c.mu.Unlock() + + // update host info + for _, h := range hosts { + h.PriceTable.HostBlockHeight = cs.BlockHeight // ignore HostBlockHeight + hi := calculateHostInfo(state.cfg, state.rs, gc, h, minScore, hostData[h.PublicKey]) + err := c.ap.bus.UpdateHostInfo(ctx, c.ap.id, h.PublicKey, hi.Gouging, hi.Score, hi.Usability) + if err != nil { + c.logger.Errorw(fmt.Sprintf("failed to update host info, err: %v", err), "hk", h.PublicKey) + } + } + return nil +} + +func (c *contractor) runContractFormations(ctx context.Context, w Worker, candidates scoredHosts, usedHosts map[types.PublicKey]struct{}, unusableHosts unusableHostsBreakdown, missing uint64, budget *types.Currency) (formed []api.ContractMetadata, _ error) { if c.ap.isStopped() { return nil, nil } @@ -1300,13 +1309,13 @@ func (c *contractor) calculateMinScore(ctx context.Context, candidates []scoredH return minScore } -func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.Host, usedHosts map[types.PublicKey]struct{}, storedData map[types.PublicKey]uint64, minScore float64) ([]scoredHost, unusableHostResult, error) { +func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.Host, usedHosts map[types.PublicKey]struct{}, storedData map[types.PublicKey]uint64, minScore float64) ([]scoredHost, unusableHostsBreakdown, error) { start := time.Now() // fetch consensus state cs, err := c.ap.bus.ConsensusState(ctx) if err != nil { - return nil, unusableHostResult{}, err + return nil, unusableHostsBreakdown{}, err } // create a gouging checker @@ -1336,7 +1345,7 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.Host, us "used", len(usedHosts)) // score all unused hosts - var unusableHostResult unusableHostResult + var unusableHosts unusableHostsBreakdown var unusable, zeros int var candidates []scoredHost for _, h := range unused { @@ -1349,15 +1358,15 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.Host, us // NOTE: ignore the pricetable's HostBlockHeight by setting it to our // own blockheight h.PriceTable.HostBlockHeight = cs.BlockHeight - usable, result := isUsableHost(state.cfg, state.rs, gc, h, minScore, storedData[h.PublicKey]) - if usable { - candidates = append(candidates, scoredHost{h, result.scoreBreakdown.Score()}) + hi := calculateHostInfo(state.cfg, state.rs, gc, h, minScore, storedData[h.PublicKey]) + if hi.Usability.Usable() { + candidates = append(candidates, scoredHost{h, hi.Score.Score()}) continue } // keep track of unusable host results - unusableHostResult.merge(result) - if result.scoreBreakdown.Score() == 0 { + unusableHosts.track(hi.Usability) + if hi.Score.Score() == 0 { zeros++ } unusable++ @@ -1368,7 +1377,7 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.Host, us "unusable", unusable, "used", len(usedHosts)) - return candidates, unusableHostResult, nil + return candidates, unusableHosts, nil } func (c *contractor) renewContract(ctx context.Context, w Worker, ci contractInfo, budget *types.Currency) (cm api.ContractMetadata, proceed bool, err error) { diff --git a/autopilot/hostfilter.go b/autopilot/hostfilter.go index 574862a97..462be4588 100644 --- a/autopilot/hostfilter.go +++ b/autopilot/hostfilter.go @@ -5,7 +5,6 @@ import ( "fmt" "math" "math/big" - "strings" rhpv2 "go.sia.tech/core/rhp/v2" rhpv3 "go.sia.tech/core/rhp/v3" @@ -32,16 +31,6 @@ const ( ) var ( - errHostBlocked = errors.New("host is blocked") - errHostNotFound = errors.New("host not found") - errHostOffline = errors.New("host is offline") - errLowScore = errors.New("host's score is below minimum") - errHostRedundantIP = errors.New("host has redundant IP") - errHostPriceGouging = errors.New("host is price gouging") - errHostNotAcceptingContracts = errors.New("host is not accepting contracts") - errHostNotCompletingScan = errors.New("host is not completing scan") - errHostNotAnnounced = errors.New("host is not announced") - errContractOutOfCollateral = errors.New("contract is out of collateral") errContractOutOfFunds = errors.New("contract is out of funds") errContractUpForRenewal = errors.New("contract is up for renewal") @@ -51,7 +40,7 @@ var ( errContractNotConfirmed = errors.New("contract hasn't been confirmed on chain in time") ) -type unusableHostResult struct { +type unusableHostsBreakdown struct { blocked uint64 offline uint64 lowscore uint64 @@ -61,99 +50,39 @@ type unusableHostResult struct { notannounced uint64 notcompletingscan uint64 unknown uint64 - - // gougingBreakdown is mostly ignored, we overload the unusableHostResult - // with a gouging breakdown to be able to return it in the host infos - // endpoint `/hosts/:hostkey` - gougingBreakdown api.HostGougingBreakdown - - // scoreBreakdown is mostly ignored, we overload the unusableHostResult with - // a score breakdown to be able to return it in the host infos endpoint - // `/hosts/:hostkey` - scoreBreakdown api.HostScoreBreakdown } -func newUnusableHostResult(errs []error, gougingBreakdown api.HostGougingBreakdown, scoreBreakdown api.HostScoreBreakdown) (u unusableHostResult) { - for _, err := range errs { - if errors.Is(err, errHostBlocked) { - u.blocked++ - } else if errors.Is(err, errHostOffline) { - u.offline++ - } else if errors.Is(err, errLowScore) { - u.lowscore++ - } else if errors.Is(err, errHostRedundantIP) { - u.redundantip++ - } else if errors.Is(err, errHostPriceGouging) { - u.gouging++ - } else if errors.Is(err, errHostNotAcceptingContracts) { - u.notacceptingcontracts++ - } else if errors.Is(err, errHostNotAnnounced) { - u.notannounced++ - } else if errors.Is(err, errHostNotCompletingScan) { - u.notcompletingscan++ - } else { - u.unknown++ - } +func (u *unusableHostsBreakdown) track(ub api.HostUsabilityBreakdown) { + if ub.Blocked { + u.blocked++ } - - u.gougingBreakdown = gougingBreakdown - u.scoreBreakdown = scoreBreakdown - return -} - -func (u unusableHostResult) String() string { - return fmt.Sprintf("host is unusable because of the following reasons: %v", strings.Join(u.reasons(), ", ")) -} - -func (u unusableHostResult) reasons() []string { - var reasons []string - if u.blocked > 0 { - reasons = append(reasons, errHostBlocked.Error()) + if ub.Offline { + u.offline++ } - if u.offline > 0 { - reasons = append(reasons, errHostOffline.Error()) + if ub.LowScore { + u.lowscore++ } - if u.lowscore > 0 { - reasons = append(reasons, errLowScore.Error()) + if ub.RedundantIP { + u.redundantip++ } - if u.redundantip > 0 { - reasons = append(reasons, errHostRedundantIP.Error()) + if ub.Gouging { + u.gouging++ } - if u.gouging > 0 { - reasons = append(reasons, errHostPriceGouging.Error()) + if ub.NotAcceptingContracts { + u.notacceptingcontracts++ } - if u.notacceptingcontracts > 0 { - reasons = append(reasons, errHostNotAcceptingContracts.Error()) + if ub.NotAnnounced { + u.notannounced++ } - if u.notannounced > 0 { - reasons = append(reasons, errHostNotAnnounced.Error()) + if ub.NotCompletingScan { + u.notcompletingscan++ } - if u.notcompletingscan > 0 { - reasons = append(reasons, errHostNotCompletingScan.Error()) + if ub.Unknown { + u.unknown++ } - if u.unknown > 0 { - reasons = append(reasons, "unknown") - } - return reasons } -func (u *unusableHostResult) merge(other unusableHostResult) { - u.blocked += other.blocked - u.offline += other.offline - u.lowscore += other.lowscore - u.redundantip += other.redundantip - u.gouging += other.gouging - u.notacceptingcontracts += other.notacceptingcontracts - u.notannounced += other.notannounced - u.notcompletingscan += other.notcompletingscan - u.unknown += other.unknown - - // scoreBreakdown is not merged - // - // gougingBreakdown is not merged -} - -func (u *unusableHostResult) keysAndValues() []interface{} { +func (u *unusableHostsBreakdown) keysAndValues() []interface{} { values := []interface{}{ "blocked", u.blocked, "offline", u.offline, @@ -174,36 +103,38 @@ func (u *unusableHostResult) keysAndValues() []interface{} { return values } -// isUsableHost returns whether the given host is usable along with a list of -// reasons why it was deemed unusable. -func isUsableHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.GougingChecker, h hostdb.Host, minScore float64, storedData uint64) (bool, unusableHostResult) { +// calculateHostInfo returns the host info for the given host. This function will +func calculateHostInfo(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.GougingChecker, h hostdb.Host, minScore float64, storedData uint64) api.HostInfo { + // sanity check redundancy settings if rs.Validate() != nil { panic("invalid redundancy settings were supplied - developer error") } - var errs []error - var gougingBreakdown api.HostGougingBreakdown - var scoreBreakdown api.HostScoreBreakdown + // prepare host breakdown fields + var ub api.HostUsabilityBreakdown + var gb api.HostGougingBreakdown + var sb api.HostScoreBreakdown + // populate host info fields if !h.IsAnnounced() { - errs = append(errs, errHostNotAnnounced) + ub.NotAnnounced = true } else if !h.Scanned { - errs = append(errs, errHostNotCompletingScan) + ub.NotCompletingScan = true } else { // online check if !h.IsOnline() { - errs = append(errs, errHostOffline) + ub.Offline = true } // accepting contracts check if !h.Settings.AcceptingContracts { - errs = append(errs, errHostNotAcceptingContracts) + ub.NotAcceptingContracts = true } // perform gouging checks - gougingBreakdown = gc.Check(&h.Settings, &h.PriceTable.HostPriceTable) - if gougingBreakdown.Gouging() { - errs = append(errs, fmt.Errorf("%w: %v", errHostPriceGouging, gougingBreakdown)) + gb = gc.Check(&h.Settings, &h.PriceTable.HostPriceTable) + if gb.Gouging() { + ub.Gouging = true } else if minScore > 0 { // perform scoring checks // @@ -211,14 +142,19 @@ func isUsableHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker. // not gouging, this because the core package does not have overflow // checks in its cost calculations needed to calculate the period // cost - scoreBreakdown = hostScore(cfg, h, storedData, rs.Redundancy()) - if scoreBreakdown.Score() < minScore { - errs = append(errs, fmt.Errorf("%w: (%s): %v < %v", errLowScore, scoreBreakdown.String(), scoreBreakdown.Score(), minScore)) + sb = hostScore(cfg, h, storedData, rs.Redundancy()) + if sb.Score() < minScore { + ub.LowScore = true } } } - return len(errs) == 0, newUnusableHostResult(errs, gougingBreakdown, scoreBreakdown) + return api.HostInfo{ + Host: h, + Usability: ub, + Gouging: gb, + Score: sb, + } } // isUsableContract returns whether the given contract is @@ -269,7 +205,7 @@ func (c *contractor) isUsableContract(cfg api.AutopilotConfig, state state, ci c // IP check should be last since it modifies the filter shouldFilter := !cfg.Hosts.AllowRedundantIPs && (usable || recoverable) if shouldFilter && f.IsRedundantIP(contract.HostIP, contract.HostKey) { - reasons = append(reasons, errHostRedundantIP.Error()) + reasons = append(reasons, api.ErrUsabilityHostRedundantIP.Error()) usable = false recoverable = false // do not use in the contract set, but keep it around for downloads renew = false // do not renew, but allow refreshes so the contracts stays funded diff --git a/autopilot/hostinfo.go b/autopilot/hostinfo.go deleted file mode 100644 index 82efa1d61..000000000 --- a/autopilot/hostinfo.go +++ /dev/null @@ -1,206 +0,0 @@ -package autopilot - -import ( - "context" - "fmt" - - "go.sia.tech/core/types" - "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" - "go.sia.tech/renterd/worker" -) - -func (c *contractor) HostInfo(ctx context.Context, hostKey types.PublicKey) (api.HostHandlerResponse, error) { - state := c.ap.State() - - if state.cfg.Contracts.Allowance.IsZero() { - return api.HostHandlerResponse{}, fmt.Errorf("can not score hosts because contracts allowance is zero") - } - if state.cfg.Contracts.Amount == 0 { - return api.HostHandlerResponse{}, fmt.Errorf("can not score hosts because contracts amount is zero") - } - if state.cfg.Contracts.Period == 0 { - return api.HostHandlerResponse{}, fmt.Errorf("can not score hosts because contract period is zero") - } - - host, err := c.ap.bus.Host(ctx, hostKey) - if err != nil { - return api.HostHandlerResponse{}, fmt.Errorf("failed to fetch requested host from bus: %w", err) - } - gs, err := c.ap.bus.GougingSettings(ctx) - if err != nil { - return api.HostHandlerResponse{}, fmt.Errorf("failed to fetch gouging settings from bus: %w", err) - } - rs, err := c.ap.bus.RedundancySettings(ctx) - if err != nil { - return api.HostHandlerResponse{}, fmt.Errorf("failed to fetch redundancy settings from bus: %w", err) - } - cs, err := c.ap.bus.ConsensusState(ctx) - if err != nil { - return api.HostHandlerResponse{}, fmt.Errorf("failed to fetch consensus state from bus: %w", err) - } - fee, err := c.ap.bus.RecommendedFee(ctx) - if err != nil { - return api.HostHandlerResponse{}, fmt.Errorf("failed to fetch recommended fee from bus: %w", err) - } - c.mu.Lock() - storedData := c.cachedDataStored[hostKey] - minScore := c.cachedMinScore - c.mu.Unlock() - - gc := worker.NewGougingChecker(gs, cs, fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) - - // ignore the pricetable's HostBlockHeight by setting it to our own blockheight - host.Host.PriceTable.HostBlockHeight = cs.BlockHeight - - isUsable, unusableResult := isUsableHost(state.cfg, rs, gc, host.Host, minScore, storedData) - return api.HostHandlerResponse{ - Host: host.Host, - Checks: &api.HostHandlerResponseChecks{ - Gouging: unusableResult.gougingBreakdown.Gouging(), - GougingBreakdown: unusableResult.gougingBreakdown, - Score: unusableResult.scoreBreakdown.Score(), - ScoreBreakdown: unusableResult.scoreBreakdown, - Usable: isUsable, - UnusableReasons: unusableResult.reasons(), - }, - }, nil -} - -func (c *contractor) hostInfoFromCache(ctx context.Context, host hostdb.Host) (hi hostInfo, found bool) { - // grab host details from cache - c.mu.Lock() - hi, found = c.cachedHostInfo[host.PublicKey] - storedData := c.cachedDataStored[host.PublicKey] - minScore := c.cachedMinScore - c.mu.Unlock() - - // return early if the host info is not cached - if !found { - return - } - - // try and refresh the host info if it got scanned in the meantime, this - // inconsistency would resolve itself but trying to update it here improves - // first time user experience - if host.Scanned && hi.UnusableResult.notcompletingscan > 0 { - cs, err := c.ap.bus.ConsensusState(ctx) - if err != nil { - c.logger.Error("failed to fetch consensus state from bus: %v", err) - } else { - state := c.ap.State() - gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) - isUsable, unusableResult := isUsableHost(state.cfg, state.rs, gc, host, minScore, storedData) - hi = hostInfo{ - Usable: isUsable, - UnusableResult: unusableResult, - } - - // update cache - c.mu.Lock() - c.cachedHostInfo[host.PublicKey] = hi - c.mu.Unlock() - } - } - - return -} - -func (c *contractor) HostInfos(ctx context.Context, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.HostHandlerResponse, error) { - // declare helper to decide whether to keep a host. - if !isValidUsabilityFilterMode(usabilityMode) { - return nil, fmt.Errorf("invalid usability mode: '%v', options are 'usable', 'unusable' or an empty string for no filter", usabilityMode) - } - - keep := func(usable bool) bool { - switch usabilityMode { - case api.UsabilityFilterModeUsable: - return usable // keep usable - case api.UsabilityFilterModeUnusable: - return !usable // keep unusable - case api.UsabilityFilterModeAll: - return true // keep all - case "": - return true // keep all - default: - panic("unreachable") - } - } - - var hostInfos []api.HostHandlerResponse - wanted := limit - for { - // fetch up to 'limit' hosts. - hosts, err := c.ap.bus.SearchHosts(ctx, api.SearchHostOptions{ - Offset: offset, - Limit: limit, - FilterMode: filterMode, - AddressContains: addressContains, - KeyIn: keyIn, - }) - if err != nil { - return nil, err - } - offset += len(hosts) - - // if there are no more hosts, we're done. - if len(hosts) == 0 { - return hostInfos, nil // no more hosts - } - - // decide how many of the returned hosts to keep. - var keptHosts int - for _, host := range hosts { - hi, cached := c.hostInfoFromCache(ctx, host) - if !cached { - // when the filterMode is "all" we include uncached hosts and - // set IsChecked = false. - if usabilityMode == api.UsabilityFilterModeAll { - hostInfos = append(hostInfos, api.HostHandlerResponse{ - Host: host, - }) - if wanted > 0 && len(hostInfos) == wanted { - return hostInfos, nil // we're done. - } - keptHosts++ - } - continue - } - if !keep(hi.Usable) { - continue - } - hostInfos = append(hostInfos, api.HostHandlerResponse{ - Host: host, - Checks: &api.HostHandlerResponseChecks{ - Gouging: hi.UnusableResult.gougingBreakdown.Gouging(), - GougingBreakdown: hi.UnusableResult.gougingBreakdown, - Score: hi.UnusableResult.scoreBreakdown.Score(), - ScoreBreakdown: hi.UnusableResult.scoreBreakdown, - Usable: hi.Usable, - UnusableReasons: hi.UnusableResult.reasons(), - }, - }) - if wanted > 0 && len(hostInfos) == wanted { - return hostInfos, nil // we're done. - } - keptHosts++ - } - - // if no hosts were kept from this batch, double the limit. - if limit > 0 && keptHosts == 0 { - limit *= 2 - } - } -} - -func isValidUsabilityFilterMode(usabilityMode string) bool { - switch usabilityMode { - case api.UsabilityFilterModeUsable: - case api.UsabilityFilterModeUnusable: - case api.UsabilityFilterModeAll: - case "": - default: - return false - } - return true -} diff --git a/bus/bus.go b/bus/bus.go index c8875a092..7df106f73 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -106,7 +106,7 @@ type ( UpdateHostBlocklistEntries(ctx context.Context, add, remove []string, clear bool) error HostInfo(ctx context.Context, autopilotID string, hk types.PublicKey) (api.HostInfo, error) - HostInfos(ctx context.Context, autopilotID string) ([]api.HostInfo, error) + HostInfos(ctx context.Context, autopilotID string, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.HostInfo, error) UpdateHostInfo(ctx context.Context, autopilotID string, hk types.PublicKey, gouging api.HostGougingBreakdown, score api.HostScoreBreakdown, usability api.HostUsabilityBreakdown) error } @@ -258,9 +258,9 @@ func (b *bus) Handler() http.Handler { "GET /autopilot/:id": b.autopilotsHandlerGET, "PUT /autopilot/:id": b.autopilotsHandlerPUT, - "GET /autopilot/:id/hosts": b.autopilotHostInfosHandlerGET, "GET /autopilot/:id/host/:hostkey": b.autopilotHostInfoHandlerGET, "PUT /autopilot/:id/host/:hostkey": b.autopilotHostInfoHandlerPUT, + "POST /autopilot/:id/hosts": b.autopilotHostInfosHandlerPOST, "GET /buckets": b.bucketsHandlerGET, "POST /buckets": b.bucketsHandlerPOST, @@ -1989,43 +1989,67 @@ func (b *bus) autopilotHostInfoHandlerGET(jc jape.Context) { jc.Encode(hi) } -func (b *bus) autopilotHostInfosHandlerGET(jc jape.Context) { +func (b *bus) autopilotHostInfoHandlerPUT(jc jape.Context) { var id string if jc.DecodeParam("id", &id) != nil { return } + var hostKey types.PublicKey + if jc.DecodeParam("hostkey", &hostKey) != nil { + return + } + var hir api.UpdateHostInfoRequest + if jc.Check("failed to decode host info", jc.Decode(&hir)) != nil { + return + } - his, err := b.hdb.HostInfos(jc.Request.Context(), id) + err := b.hdb.UpdateHostInfo(jc.Request.Context(), id, hostKey, hir.Gouging, hir.Score, hir.Usability) if errors.Is(err, api.ErrAutopilotNotFound) { jc.Error(err, http.StatusNotFound) return - } else if jc.Check("failed to fetch host infos", err) != nil { + } else if jc.Check("failed to update host info", err) != nil { return } - jc.Encode(his) } -func (b *bus) autopilotHostInfoHandlerPUT(jc jape.Context) { +func (b *bus) autopilotHostInfosHandlerPOST(jc jape.Context) { var id string if jc.DecodeParam("id", &id) != nil { return } - var hostKey types.PublicKey - if jc.DecodeParam("hostkey", &hostKey) != nil { + var req api.HostInfosRequest + if jc.Decode(&req) != nil { return } - var hir api.UpdateHostInfoRequest - if jc.Check("failed to decode host info", jc.Decode(&hir)) != nil { - return + + // validate filter mode + if fm := req.FilterMode; fm != "" { + if fm != api.HostFilterModeAll && + fm != api.HostFilterModeAllowed && + fm != api.HostFilterModeBlocked { + jc.Error(fmt.Errorf("invalid filter mode: '%v', allowed values are '%s', '%s', '%s'", fm, api.HostFilterModeAll, api.HostFilterModeAllowed, api.HostFilterModeBlocked), http.StatusBadRequest) + return + } } - err := b.hdb.UpdateHostInfo(jc.Request.Context(), id, hostKey, hir.Gouging, hir.Score, hir.Usability) + // validate usability mode + if um := req.UsabilityMode; um != "" { + if um != api.UsabilityFilterModeUsable && + um != api.UsabilityFilterModeUnusable && + um != api.UsabilityFilterModeAll { + jc.Error(fmt.Errorf("invalid usability mode: '%v', allowed values are '%s', '%s', '%s'", um, api.UsabilityFilterModeAll, api.UsabilityFilterModeUsable, api.UsabilityFilterModeUnusable), http.StatusBadRequest) + return + } + } + + his, err := b.hdb.HostInfos(jc.Request.Context(), id, req.FilterMode, req.UsabilityMode, req.AddressContains, req.KeyIn, req.Offset, req.Limit) if errors.Is(err, api.ErrAutopilotNotFound) { jc.Error(err, http.StatusNotFound) return - } else if jc.Check("failed to update host info", err) != nil { + } else if jc.Check("failed to fetch host infos", err) != nil { return } + jc.Encode(his) } func (b *bus) contractTaxHandlerGET(jc jape.Context) { diff --git a/bus/client/hosts.go b/bus/client/hosts.go index ecf44e52b..beda5950b 100644 --- a/bus/client/hosts.go +++ b/bus/client/hosts.go @@ -100,3 +100,32 @@ func (c *Client) UpdateHostBlocklist(ctx context.Context, add, remove []string, err = c.c.WithContext(ctx).PUT("/hosts/blocklist", api.UpdateBlocklistRequest{Add: add, Remove: remove, Clear: clear}) return } + +// HostInfo returns the host info for a given host and autopilot identifier. +func (c *Client) HostInfo(ctx context.Context, autopilotID string, hostKey types.PublicKey) (hostInfo api.HostInfo, err error) { + err = c.c.WithContext(ctx).GET(fmt.Sprintf("/autopilot/%s/host/%s", autopilotID, hostKey), &hostInfo) + return +} + +// UpdateHostInfo updates the host info for a given host and autopilot identifier. +func (c *Client) UpdateHostInfo(ctx context.Context, autopilotID string, hostKey types.PublicKey, gouging api.HostGougingBreakdown, score api.HostScoreBreakdown, usability api.HostUsabilityBreakdown) (err error) { + err = c.c.WithContext(ctx).PUT(fmt.Sprintf("/autopilot/%s/host/%s", autopilotID, hostKey), api.UpdateHostInfoRequest{ + Gouging: gouging, + Score: score, + Usability: usability, + }) + return +} + +// HostInfos returns the host info for all hosts known to the autopilot with the given identifier. +func (c *Client) HostInfos(ctx context.Context, autopilotID string, opts api.HostInfoOptions) (hostInfos []api.HostInfo, err error) { + err = c.c.WithContext(ctx).POST(fmt.Sprintf("/autopilot/%s", autopilotID), api.HostInfosRequest{ + Offset: opts.Offset, + Limit: opts.Limit, + FilterMode: opts.FilterMode, + UsabilityMode: opts.UsabilityMode, + AddressContains: opts.AddressContains, + KeyIn: opts.KeyIn, + }, &hostInfos) + return +} diff --git a/stores/hostdb.go b/stores/hostdb.go index 0a293f81f..0bc1e5a8c 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -596,7 +596,11 @@ func (ss *SQLStore) HostInfo(ctx context.Context, autopilotID string, hk types.P return } -func (ss *SQLStore) HostInfos(ctx context.Context, autopilotID string) (his []api.HostInfo, err error) { +func (ss *SQLStore) HostInfos(ctx context.Context, autopilotID string, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) (his []api.HostInfo, err error) { + if offset < 0 { + return nil, ErrNegativeOffset + } + err = ss.db.Transaction(func(tx *gorm.DB) error { // fetch ap id var apID uint @@ -611,12 +615,61 @@ func (ss *SQLStore) HostInfos(ctx context.Context, autopilotID string) (his []ap return err } - // fetch host info - var infos []dbHostInfo - if err := tx. + // prepare query + query := tx. Model(&dbHostInfo{}). Where("db_autopilot_id = ?", apID). - Preload("DBHost"). + Joins("DBHost") + + // apply mode filter + switch filterMode { + case api.HostFilterModeAllowed: + query = query.Scopes(ss.excludeBlocked("DBHost")) + case api.HostFilterModeBlocked: + query = query.Scopes(ss.excludeAllowed("DBHost")) + case api.HostFilterModeAll: + // nothing to do + default: + return fmt.Errorf("invalid filter mode: %v", filterMode) + } + + // apply usability filter + switch usabilityMode { + case api.UsabilityFilterModeUsable: + query = query.Where("usability_blocked = ? AND usability_offline = ? AND usability_low_score = ? AND usability_redundant_ip = ? AND usability_gouging = ? AND usability_not_accepting_contracts = ? AND usability_not_announced = ? AND usability_not_completing_scan = ? AND usability_unknown = ?", + false, false, false, false, false, false, false, false, false) + case api.UsabilityFilterModeUnusable: + query = query.Where("usability_blocked = ? OR usability_offline = ? OR usability_low_score = ? OR usability_redundant_ip = ? OR usability_gouging = ? OR usability_not_accepting_contracts = ? OR usability_not_announced = ? OR usability_not_completing_scan = ? OR usability_unknown = ?", + true, true, true, true, true, true, true, true, true) + case api.UsabilityFilterModeAll: + // nothing to do + default: + return fmt.Errorf("invalid usability mode: %v", usabilityMode) + } + + // apply address filter + if addressContains != "" { + query = query.Scopes(func(d *gorm.DB) *gorm.DB { + return d.Where("net_address LIKE ?", "%"+addressContains+"%") + }) + } + + // apply key filter + if len(keyIn) > 0 { + pubKeys := make([]publicKey, len(keyIn)) + for i, pk := range keyIn { + pubKeys[i] = publicKey(pk) + } + query = query.Scopes(func(d *gorm.DB) *gorm.DB { + return d.Where("public_key IN ?", pubKeys) + }) + } + + // fetch host info + var infos []dbHostInfo + if err := query. + Offset(offset). + Limit(limit). Find(&infos). Error; err != nil { return err @@ -715,9 +768,9 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains query := ss.db switch filterMode { case api.HostFilterModeAllowed: - query = query.Scopes(ss.excludeBlocked) + query = query.Scopes(ss.excludeBlocked("hosts")) case api.HostFilterModeBlocked: - query = query.Scopes(ss.excludeAllowed) + query = query.Scopes(ss.excludeAllowed("hosts")) case api.HostFilterModeAll: // nothing to do default: @@ -1157,37 +1210,41 @@ func (ss *SQLStore) processConsensusChangeHostDB(cc modules.ConsensusChange) { // excludeBlocked can be used as a scope for a db transaction to exclude blocked // hosts. -func (ss *SQLStore) excludeBlocked(db *gorm.DB) *gorm.DB { - ss.mu.Lock() - defer ss.mu.Unlock() +func (ss *SQLStore) excludeBlocked(alias string) func(db *gorm.DB) *gorm.DB { + return func(db *gorm.DB) *gorm.DB { + ss.mu.Lock() + defer ss.mu.Unlock() - if ss.hasAllowlist { - db = db.Where("EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") - } - if ss.hasBlocklist { - db = db.Where("NOT EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") + if ss.hasAllowlist { + db = db.Where(fmt.Sprintf("EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", alias)) + } + if ss.hasBlocklist { + db = db.Where(fmt.Sprintf("NOT EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", alias)) + } + return db } - return db } // excludeAllowed can be used as a scope for a db transaction to exclude allowed // hosts. -func (ss *SQLStore) excludeAllowed(db *gorm.DB) *gorm.DB { - ss.mu.Lock() - defer ss.mu.Unlock() +func (ss *SQLStore) excludeAllowed(alias string) func(db *gorm.DB) *gorm.DB { + return func(db *gorm.DB) *gorm.DB { + ss.mu.Lock() + defer ss.mu.Unlock() - if ss.hasAllowlist { - db = db.Where("NOT EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") - } - if ss.hasBlocklist { - db = db.Where("EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") - } - if !ss.hasAllowlist && !ss.hasBlocklist { - // if neither an allowlist nor a blocklist exist, all hosts are allowed - // which means we return none - db = db.Where("1 = 0") + if ss.hasAllowlist { + db = db.Where(fmt.Sprintf("NOT EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", alias)) + } + if ss.hasBlocklist { + db = db.Where(fmt.Sprintf("EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", alias)) + } + if !ss.hasAllowlist && !ss.hasBlocklist { + // if neither an allowlist nor a blocklist exist, all hosts are allowed + // which means we return none + db = db.Where("1 = 0") + } + return db } - return db } func (ss *SQLStore) isBlocked(h dbHost) (blocked bool) { diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index 06692ad8c..750ae65f5 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "os" "reflect" "testing" "time" @@ -1065,7 +1066,11 @@ func TestAnnouncementMaxAge(t *testing.T) { } func TestHostInfo(t *testing.T) { - ss := newTestSQLStore(t, defaultTestSQLStoreConfig) + cfg := defaultTestSQLStoreConfig + cfg.persistent = true + cfg.dir = "/Users/peterjan/testing" + os.RemoveAll(cfg.dir) + ss := newTestSQLStore(t, cfg) defer ss.Close() // fetch info for a non-existing autopilot @@ -1133,7 +1138,7 @@ func TestHostInfo(t *testing.T) { } // add another host info - err = ss.addTestHost(types.PublicKey{2}) + err = ss.addCustomTestHost(types.PublicKey{2}, "bar.com:1000") if err != nil { t.Fatal(err) } @@ -1143,7 +1148,7 @@ func TestHostInfo(t *testing.T) { } // fetch all infos for autopilot - his, err := ss.HostInfos(context.Background(), "foo") + his, err := ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 0, -1) if err != nil { t.Fatal(err) } else if len(his) != 2 { @@ -1151,6 +1156,102 @@ func TestHostInfo(t *testing.T) { } else if his[0].Host.PublicKey != (types.PublicKey{1}) || his[1].Host.PublicKey != (types.PublicKey{2}) { t.Fatal("unexpected", his) } + + // fetch infos using offset & limit + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 0, 1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 1, 1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 2, 1) + if err != nil { + t.Fatal(err) + } else if len(his) != 0 { + t.Fatal("unexpected") + } + + // fetch infos using net addresses + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "bar", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } else if his[0].Host.PublicKey != (types.PublicKey{2}) { + t.Fatal("unexpected", his) + } + + // fetch infos using keyIn + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", []types.PublicKey{{2}}, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } else if his[0].Host.PublicKey != (types.PublicKey{2}) { + t.Fatal("unexpected", his) + } + + // fetch infos using mode filters + err = ss.UpdateHostBlocklistEntries(context.Background(), []string{"bar.com:1000"}, nil, false) + if err != nil { + t.Fatal(err) + } + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAllowed, api.UsabilityFilterModeAll, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } else if his[0].Host.PublicKey != (types.PublicKey{1}) { + t.Fatal("unexpected", his) + } + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeBlocked, api.UsabilityFilterModeAll, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } else if his[0].Host.PublicKey != (types.PublicKey{2}) { + t.Fatal("unexpected", his) + } + err = ss.UpdateHostBlocklistEntries(context.Background(), nil, nil, true) + if err != nil { + t.Fatal(err) + } + + // fetch infos using usability filters + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeUsable, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 0 { + t.Fatal("unexpected") + } + // update info + want.Usability.Blocked = false + want.Usability.Offline = false + want.Usability.LowScore = false + want.Usability.RedundantIP = false + want.Usability.Gouging = false + want.Usability.NotAcceptingContracts = false + want.Usability.NotAnnounced = false + want.Usability.NotCompletingScan = false + want.Usability.Unknown = false + err = ss.UpdateHostInfo(context.Background(), "foo", types.PublicKey{1}, want.Gouging, want.Score, want.Usability) + if err != nil { + t.Fatal(err) + } + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeUsable, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } else if his[0].Host.PublicKey != (types.PublicKey{1}) { + t.Fatal("unexpected", his) + } } // addTestHosts adds 'n' hosts to the db and returns their keys. diff --git a/worker/upload.go b/worker/upload.go index ab84e2b37..1a5e230c0 100644 --- a/worker/upload.go +++ b/worker/upload.go @@ -276,7 +276,6 @@ func (w *worker) threadedUploadPackedSlabs(rs api.RedundancySettings, contractSe // wait for all threads to finish wg.Wait() - return } func (w *worker) tryUploadPackedSlab(ctx context.Context, mem Memory, ps api.PackedSlab, rs api.RedundancySettings, contractSet string, lockPriority int) error { From f0ce1c81cab0f562bc59a028c1221325eaeb09e5 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 19 Mar 2024 08:49:05 +0100 Subject: [PATCH 03/23] autopilot: add compatV105HostInfo --- api/autopilot.go | 15 --- api/host.go | 31 +++++++ autopilot/autopilot.go | 124 +++++++++++++++++-------- autopilot/client.go | 4 +- autopilot/hostinfo.go | 206 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 324 insertions(+), 56 deletions(-) create mode 100644 autopilot/hostinfo.go diff --git a/api/autopilot.go b/api/autopilot.go index b20696fa9..d40f84ee4 100644 --- a/api/autopilot.go +++ b/api/autopilot.go @@ -119,19 +119,4 @@ type ( } Recommendation *ConfigRecommendation `json:"recommendation,omitempty"` } - - // HostHandlerResponse is the response type for the /host/:hostkey endpoint. - HostHandlerResponse struct { - Host hostdb.Host `json:"host"` - Checks *HostHandlerResponseChecks `json:"checks,omitempty"` - } - - HostHandlerResponseChecks struct { - Gouging bool `json:"gouging"` - GougingBreakdown HostGougingBreakdown `json:"gougingBreakdown"` - Score float64 `json:"score"` - ScoreBreakdown HostScoreBreakdown `json:"scoreBreakdown"` - Usable bool `json:"usable"` - UnusableReasons []string `json:"unusableReasons"` - } ) diff --git a/api/host.go b/api/host.go index 18ae2cc38..11767979c 100644 --- a/api/host.go +++ b/api/host.go @@ -74,6 +74,23 @@ type ( // endpoint. HostInfosRequest SearchHostsRequest + // HostInfoResponse is the response type for the /host/:hostkey endpoint. + // + // TODO: on next major release consider returning an api.HostInfo + HostInfoResponse struct { + Host hostdb.Host `json:"host"` + Checks *HostChecks `json:"checks,omitempty"` + } + + HostChecks struct { + Gouging bool `json:"gouging"` + GougingBreakdown HostGougingBreakdown `json:"gougingBreakdown"` + Score float64 `json:"score"` + ScoreBreakdown HostScoreBreakdown `json:"scoreBreakdown"` + Usable bool `json:"usable"` + UnusableReasons []string `json:"unusableReasons"` + } + // UpdateHostInfoRequest is the request type for the PUT // /autopilot/:id/host/:hostkey endpoint. UpdateHostInfoRequest struct { @@ -266,6 +283,20 @@ func (ub HostUsabilityBreakdown) UnusableReasons() []string { return reasons } +func (hi HostInfo) ToHostInfoReponse() HostInfoResponse { + return HostInfoResponse{ + Host: hi.Host, + Checks: &HostChecks{ + Gouging: hi.Usability.Gouging, + GougingBreakdown: hi.Gouging, + Score: hi.Score.Score(), + ScoreBreakdown: hi.Score, + Usable: hi.Usability.Usable(), + UnusableReasons: hi.Usability.UnusableReasons(), + }, + } +} + func (c AutopilotConfig) Validate() error { if c.Hosts.MaxDowntimeHours > 99*365*24 { return ErrMaxDowntimeHoursTooHigh diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 608e67463..32cccf3d5 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -684,16 +684,56 @@ func (ap *Autopilot) triggerHandlerPOST(jc jape.Context) { } func (ap *Autopilot) hostHandlerGET(jc jape.Context) { - var hostKey types.PublicKey - if jc.DecodeParam("hostKey", &hostKey) != nil { + var hk types.PublicKey + if jc.DecodeParam("hostKey", &hk) != nil { return } - host, err := ap.bus.HostInfo(jc.Request.Context(), ap.id, hostKey) + // TODO: remove on next major release + h, err := compatV105HostInfo(jc.Request.Context(), ap.State(), ap.bus, hk) if jc.Check("failed to get host info", err) != nil { return } - jc.Encode(host) + + hi, err := ap.bus.HostInfo(jc.Request.Context(), ap.id, hk) + if utils.IsErr(err, api.ErrHostInfoNotFound) { + // TODO PJ: we used to calculate the host info here on the fly, maybe we + // should keep doing that but maybe we can get away with this too... + jc.Encode(api.HostInfoResponse{ + Host: h.Host, + Checks: nil, + }) + return + } else if jc.Check("failed to get host info", err) != nil { + return + } + + jc.Encode(hi.ToHostInfoReponse()) +} + +func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { + var req api.HostInfosRequest + if jc.Decode(&req) != nil { + return + } + hosts, err := ap.bus.HostInfos(jc.Request.Context(), ap.id, api.HostInfoOptions{ + UsabilityMode: req.UsabilityMode, + SearchHostOptions: api.SearchHostOptions{ + FilterMode: req.FilterMode, + AddressContains: req.AddressContains, + KeyIn: req.KeyIn, + Offset: req.Offset, + Limit: req.Limit, + }, + }) + if jc.Check("failed to get host info", err) != nil { + return + } + resps := make([]api.HostInfoResponse, len(hosts)) + for i, host := range hosts { + resps[i] = host.ToHostInfoReponse() + } + jc.Encode(resps) } func (ap *Autopilot) stateHandlerGET(jc jape.Context) { @@ -727,41 +767,6 @@ func (ap *Autopilot) stateHandlerGET(jc jape.Context) { }) } -func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { - var req api.HostInfosRequest - if jc.Decode(&req) != nil { - return - } - hosts, err := ap.bus.HostInfos(jc.Request.Context(), ap.id, api.HostInfoOptions{ - UsabilityMode: req.UsabilityMode, - SearchHostOptions: api.SearchHostOptions{ - FilterMode: req.FilterMode, - AddressContains: req.AddressContains, - KeyIn: req.KeyIn, - Offset: req.Offset, - Limit: req.Limit, - }, - }) - if jc.Check("failed to get host info", err) != nil { - return - } - resps := make([]api.HostHandlerResponse, len(hosts)) - for i, host := range hosts { - resps[i] = api.HostHandlerResponse{ - Host: host.Host, - Checks: &api.HostHandlerResponseChecks{ - Gouging: host.Gouging.Gouging(), - GougingBreakdown: host.Gouging, - Score: host.Score.Score(), - ScoreBreakdown: host.Score, - Usable: host.Usability.Usable(), - UnusableReasons: host.Usability.UnusableReasons(), - }, - } - } - jc.Encode(resps) -} - func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.Host) (usables uint64) { gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow) for _, host := range hosts { @@ -930,3 +935,44 @@ func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg nSteps++ } } + +// compatV105HostInfo performs some state checks and bus calls we no longer +// need, but are necessary checks to make sure our API is consistent. This +// should be considered for removal when releasing a new major version. +func compatV105HostInfo(ctx context.Context, s state, b Bus, hk types.PublicKey) (*hostdb.HostInfo, error) { + // state checks + if s.cfg.Contracts.Allowance.IsZero() { + return nil, fmt.Errorf("can not score hosts because contracts allowance is zero") + } + if s.cfg.Contracts.Amount == 0 { + return nil, fmt.Errorf("can not score hosts because contracts amount is zero") + } + if s.cfg.Contracts.Period == 0 { + return nil, fmt.Errorf("can not score hosts because contract period is zero") + } + + // fetch host + host, err := b.Host(ctx, hk) + if err != nil { + return nil, fmt.Errorf("failed to fetch requested host from bus: %w", err) + } + + // other checks + _, err = b.GougingSettings(ctx) + if err != nil { + return nil, fmt.Errorf("failed to fetch gouging settings from bus: %w", err) + } + _, err = b.RedundancySettings(ctx) + if err != nil { + return nil, fmt.Errorf("failed to fetch redundancy settings from bus: %w", err) + } + _, err = b.ConsensusState(ctx) + if err != nil { + return nil, fmt.Errorf("failed to fetch consensus state from bus: %w", err) + } + _, err = b.RecommendedFee(ctx) + if err != nil { + return nil, fmt.Errorf("failed to fetch recommended fee from bus: %w", err) + } + return &host, nil +} diff --git a/autopilot/client.go b/autopilot/client.go index ba16754a5..41b0b3207 100644 --- a/autopilot/client.go +++ b/autopilot/client.go @@ -34,13 +34,13 @@ func (c *Client) UpdateConfig(cfg api.AutopilotConfig) error { } // HostInfo returns information about the host with given host key. -func (c *Client) HostInfo(hostKey types.PublicKey) (resp api.HostHandlerResponse, err error) { +func (c *Client) HostInfo(hostKey types.PublicKey) (resp api.HostInfoResponse, err error) { err = c.c.GET(fmt.Sprintf("/host/%s", hostKey), &resp) return } // HostInfo returns information about all hosts. -func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode string, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostHandlerResponse, err error) { +func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode string, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostInfoResponse, err error) { err = c.c.POST("/hosts", api.SearchHostsRequest{ Offset: offset, Limit: limit, diff --git a/autopilot/hostinfo.go b/autopilot/hostinfo.go new file mode 100644 index 000000000..263cd5d4b --- /dev/null +++ b/autopilot/hostinfo.go @@ -0,0 +1,206 @@ +package autopilot + +import ( + "context" + "fmt" + + "go.sia.tech/core/types" + "go.sia.tech/renterd/api" + "go.sia.tech/renterd/hostdb" + "go.sia.tech/renterd/worker" +) + +func (c *contractor) HostInfo(ctx context.Context, hostKey types.PublicKey) (api.HostInfoResponse, error) { + state := c.ap.State() + + if state.cfg.Contracts.Allowance.IsZero() { + return api.HostInfoResponse{}, fmt.Errorf("can not score hosts because contracts allowance is zero") + } + if state.cfg.Contracts.Amount == 0 { + return api.HostInfoResponse{}, fmt.Errorf("can not score hosts because contracts amount is zero") + } + if state.cfg.Contracts.Period == 0 { + return api.HostInfoResponse{}, fmt.Errorf("can not score hosts because contract period is zero") + } + + host, err := c.ap.bus.Host(ctx, hostKey) + if err != nil { + return api.HostInfoResponse{}, fmt.Errorf("failed to fetch requested host from bus: %w", err) + } + gs, err := c.ap.bus.GougingSettings(ctx) + if err != nil { + return api.HostInfoResponse{}, fmt.Errorf("failed to fetch gouging settings from bus: %w", err) + } + rs, err := c.ap.bus.RedundancySettings(ctx) + if err != nil { + return api.HostInfoResponse{}, fmt.Errorf("failed to fetch redundancy settings from bus: %w", err) + } + cs, err := c.ap.bus.ConsensusState(ctx) + if err != nil { + return api.HostInfoResponse{}, fmt.Errorf("failed to fetch consensus state from bus: %w", err) + } + fee, err := c.ap.bus.RecommendedFee(ctx) + if err != nil { + return api.HostInfoResponse{}, fmt.Errorf("failed to fetch recommended fee from bus: %w", err) + } + c.mu.Lock() + storedData := c.cachedDataStored[hostKey] + minScore := c.cachedMinScore + c.mu.Unlock() + + gc := worker.NewGougingChecker(gs, cs, fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) + + // ignore the pricetable's HostBlockHeight by setting it to our own blockheight + host.Host.PriceTable.HostBlockHeight = cs.BlockHeight + + isUsable, unusableResult := isUsableHost(state.cfg, rs, gc, host.Host, minScore, storedData) + return api.HostInfoResponse{ + Host: host.Host, + Checks: &api.HostChecks{ + Gouging: unusableResult.gougingBreakdown.Gouging(), + GougingBreakdown: unusableResult.gougingBreakdown, + Score: unusableResult.scoreBreakdown.Score(), + ScoreBreakdown: unusableResult.scoreBreakdown, + Usable: isUsable, + UnusableReasons: unusableResult.reasons(), + }, + }, nil +} + +func (c *contractor) hostInfoFromCache(ctx context.Context, host hostdb.Host) (hi hostInfo, found bool) { + // grab host details from cache + c.mu.Lock() + hi, found = c.cachedHostInfo[host.PublicKey] + storedData := c.cachedDataStored[host.PublicKey] + minScore := c.cachedMinScore + c.mu.Unlock() + + // return early if the host info is not cached + if !found { + return + } + + // try and refresh the host info if it got scanned in the meantime, this + // inconsistency would resolve itself but trying to update it here improves + // first time user experience + if host.Scanned && hi.UnusableResult.notcompletingscan > 0 { + cs, err := c.ap.bus.ConsensusState(ctx) + if err != nil { + c.logger.Error("failed to fetch consensus state from bus: %v", err) + } else { + state := c.ap.State() + gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) + isUsable, unusableResult := isUsableHost(state.cfg, state.rs, gc, host, minScore, storedData) + hi = hostInfo{ + Usable: isUsable, + UnusableResult: unusableResult, + } + + // update cache + c.mu.Lock() + c.cachedHostInfo[host.PublicKey] = hi + c.mu.Unlock() + } + } + + return +} + +func (c *contractor) HostInfos(ctx context.Context, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.HostInfoResponse, error) { + // declare helper to decide whether to keep a host. + if !isValidUsabilityFilterMode(usabilityMode) { + return nil, fmt.Errorf("invalid usability mode: '%v', options are 'usable', 'unusable' or an empty string for no filter", usabilityMode) + } + + keep := func(usable bool) bool { + switch usabilityMode { + case api.UsabilityFilterModeUsable: + return usable // keep usable + case api.UsabilityFilterModeUnusable: + return !usable // keep unusable + case api.UsabilityFilterModeAll: + return true // keep all + case "": + return true // keep all + default: + panic("unreachable") + } + } + + var hostInfos []api.HostInfoResponse + wanted := limit + for { + // fetch up to 'limit' hosts. + hosts, err := c.ap.bus.SearchHosts(ctx, api.SearchHostOptions{ + Offset: offset, + Limit: limit, + FilterMode: filterMode, + AddressContains: addressContains, + KeyIn: keyIn, + }) + if err != nil { + return nil, err + } + offset += len(hosts) + + // if there are no more hosts, we're done. + if len(hosts) == 0 { + return hostInfos, nil // no more hosts + } + + // decide how many of the returned hosts to keep. + var keptHosts int + for _, host := range hosts { + hi, cached := c.hostInfoFromCache(ctx, host) + if !cached { + // when the filterMode is "all" we include uncached hosts and + // set IsChecked = false. + if usabilityMode == api.UsabilityFilterModeAll { + hostInfos = append(hostInfos, api.HostInfoResponse{ + Host: host, + }) + if wanted > 0 && len(hostInfos) == wanted { + return hostInfos, nil // we're done. + } + keptHosts++ + } + continue + } + if !keep(hi.Usable) { + continue + } + hostInfos = append(hostInfos, api.HostInfoResponse{ + Host: host, + Checks: &api.HostChecks{ + Gouging: hi.UnusableResult.gougingBreakdown.Gouging(), + GougingBreakdown: hi.UnusableResult.gougingBreakdown, + Score: hi.UnusableResult.scoreBreakdown.Score(), + ScoreBreakdown: hi.UnusableResult.scoreBreakdown, + Usable: hi.Usable, + UnusableReasons: hi.UnusableResult.reasons(), + }, + }) + if wanted > 0 && len(hostInfos) == wanted { + return hostInfos, nil // we're done. + } + keptHosts++ + } + + // if no hosts were kept from this batch, double the limit. + if limit > 0 && keptHosts == 0 { + limit *= 2 + } + } +} + +func isValidUsabilityFilterMode(usabilityMode string) bool { + switch usabilityMode { + case api.UsabilityFilterModeUsable: + case api.UsabilityFilterModeUnusable: + case api.UsabilityFilterModeAll: + case "": + default: + return false + } + return true +} From 8efc718a01d53ab1ec948d50b9f1c0eaea7a2a84 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 19 Mar 2024 09:11:08 +0100 Subject: [PATCH 04/23] autopilot: add compatV105UsabilityFilterModeCheck --- api/autopilot.go | 1 - autopilot/autopilot.go | 21 +++++ autopilot/hostinfo.go | 206 ----------------------------------------- stores/hostdb.go | 1 + stores/hostdb_test.go | 1 + 5 files changed, 23 insertions(+), 207 deletions(-) delete mode 100644 autopilot/hostinfo.go diff --git a/api/autopilot.go b/api/autopilot.go index d40f84ee4..70914afc3 100644 --- a/api/autopilot.go +++ b/api/autopilot.go @@ -4,7 +4,6 @@ import ( "errors" "go.sia.tech/core/types" - "go.sia.tech/renterd/hostdb" ) const ( diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 32cccf3d5..c8df13279 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -716,6 +716,15 @@ func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { if jc.Decode(&req) != nil { return } + + // TODO: remove on next major release + if jc.Check("failed to get host info", compatV105UsabilityFilterModeCheck(req.UsabilityMode)) != nil { + return + } + + // TODO PJ: we used to return hosts regardless of whether they have host + // info if usability mode was set to "all" - it is annoying but maybe we + // should keep doing that hosts, err := ap.bus.HostInfos(jc.Request.Context(), ap.id, api.HostInfoOptions{ UsabilityMode: req.UsabilityMode, SearchHostOptions: api.SearchHostOptions{ @@ -976,3 +985,15 @@ func compatV105HostInfo(ctx context.Context, s state, b Bus, hk types.PublicKey) } return &host, nil } + +func compatV105UsabilityFilterModeCheck(usabilityMode string) error { + switch usabilityMode { + case api.UsabilityFilterModeUsable: + case api.UsabilityFilterModeUnusable: + case api.UsabilityFilterModeAll: + case "": + default: + return fmt.Errorf("invalid usability mode: '%v', options are 'usable', 'unusable' or an empty string for no filter", usabilityMode) + } + return nil +} diff --git a/autopilot/hostinfo.go b/autopilot/hostinfo.go deleted file mode 100644 index 263cd5d4b..000000000 --- a/autopilot/hostinfo.go +++ /dev/null @@ -1,206 +0,0 @@ -package autopilot - -import ( - "context" - "fmt" - - "go.sia.tech/core/types" - "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" - "go.sia.tech/renterd/worker" -) - -func (c *contractor) HostInfo(ctx context.Context, hostKey types.PublicKey) (api.HostInfoResponse, error) { - state := c.ap.State() - - if state.cfg.Contracts.Allowance.IsZero() { - return api.HostInfoResponse{}, fmt.Errorf("can not score hosts because contracts allowance is zero") - } - if state.cfg.Contracts.Amount == 0 { - return api.HostInfoResponse{}, fmt.Errorf("can not score hosts because contracts amount is zero") - } - if state.cfg.Contracts.Period == 0 { - return api.HostInfoResponse{}, fmt.Errorf("can not score hosts because contract period is zero") - } - - host, err := c.ap.bus.Host(ctx, hostKey) - if err != nil { - return api.HostInfoResponse{}, fmt.Errorf("failed to fetch requested host from bus: %w", err) - } - gs, err := c.ap.bus.GougingSettings(ctx) - if err != nil { - return api.HostInfoResponse{}, fmt.Errorf("failed to fetch gouging settings from bus: %w", err) - } - rs, err := c.ap.bus.RedundancySettings(ctx) - if err != nil { - return api.HostInfoResponse{}, fmt.Errorf("failed to fetch redundancy settings from bus: %w", err) - } - cs, err := c.ap.bus.ConsensusState(ctx) - if err != nil { - return api.HostInfoResponse{}, fmt.Errorf("failed to fetch consensus state from bus: %w", err) - } - fee, err := c.ap.bus.RecommendedFee(ctx) - if err != nil { - return api.HostInfoResponse{}, fmt.Errorf("failed to fetch recommended fee from bus: %w", err) - } - c.mu.Lock() - storedData := c.cachedDataStored[hostKey] - minScore := c.cachedMinScore - c.mu.Unlock() - - gc := worker.NewGougingChecker(gs, cs, fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) - - // ignore the pricetable's HostBlockHeight by setting it to our own blockheight - host.Host.PriceTable.HostBlockHeight = cs.BlockHeight - - isUsable, unusableResult := isUsableHost(state.cfg, rs, gc, host.Host, minScore, storedData) - return api.HostInfoResponse{ - Host: host.Host, - Checks: &api.HostChecks{ - Gouging: unusableResult.gougingBreakdown.Gouging(), - GougingBreakdown: unusableResult.gougingBreakdown, - Score: unusableResult.scoreBreakdown.Score(), - ScoreBreakdown: unusableResult.scoreBreakdown, - Usable: isUsable, - UnusableReasons: unusableResult.reasons(), - }, - }, nil -} - -func (c *contractor) hostInfoFromCache(ctx context.Context, host hostdb.Host) (hi hostInfo, found bool) { - // grab host details from cache - c.mu.Lock() - hi, found = c.cachedHostInfo[host.PublicKey] - storedData := c.cachedDataStored[host.PublicKey] - minScore := c.cachedMinScore - c.mu.Unlock() - - // return early if the host info is not cached - if !found { - return - } - - // try and refresh the host info if it got scanned in the meantime, this - // inconsistency would resolve itself but trying to update it here improves - // first time user experience - if host.Scanned && hi.UnusableResult.notcompletingscan > 0 { - cs, err := c.ap.bus.ConsensusState(ctx) - if err != nil { - c.logger.Error("failed to fetch consensus state from bus: %v", err) - } else { - state := c.ap.State() - gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) - isUsable, unusableResult := isUsableHost(state.cfg, state.rs, gc, host, minScore, storedData) - hi = hostInfo{ - Usable: isUsable, - UnusableResult: unusableResult, - } - - // update cache - c.mu.Lock() - c.cachedHostInfo[host.PublicKey] = hi - c.mu.Unlock() - } - } - - return -} - -func (c *contractor) HostInfos(ctx context.Context, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.HostInfoResponse, error) { - // declare helper to decide whether to keep a host. - if !isValidUsabilityFilterMode(usabilityMode) { - return nil, fmt.Errorf("invalid usability mode: '%v', options are 'usable', 'unusable' or an empty string for no filter", usabilityMode) - } - - keep := func(usable bool) bool { - switch usabilityMode { - case api.UsabilityFilterModeUsable: - return usable // keep usable - case api.UsabilityFilterModeUnusable: - return !usable // keep unusable - case api.UsabilityFilterModeAll: - return true // keep all - case "": - return true // keep all - default: - panic("unreachable") - } - } - - var hostInfos []api.HostInfoResponse - wanted := limit - for { - // fetch up to 'limit' hosts. - hosts, err := c.ap.bus.SearchHosts(ctx, api.SearchHostOptions{ - Offset: offset, - Limit: limit, - FilterMode: filterMode, - AddressContains: addressContains, - KeyIn: keyIn, - }) - if err != nil { - return nil, err - } - offset += len(hosts) - - // if there are no more hosts, we're done. - if len(hosts) == 0 { - return hostInfos, nil // no more hosts - } - - // decide how many of the returned hosts to keep. - var keptHosts int - for _, host := range hosts { - hi, cached := c.hostInfoFromCache(ctx, host) - if !cached { - // when the filterMode is "all" we include uncached hosts and - // set IsChecked = false. - if usabilityMode == api.UsabilityFilterModeAll { - hostInfos = append(hostInfos, api.HostInfoResponse{ - Host: host, - }) - if wanted > 0 && len(hostInfos) == wanted { - return hostInfos, nil // we're done. - } - keptHosts++ - } - continue - } - if !keep(hi.Usable) { - continue - } - hostInfos = append(hostInfos, api.HostInfoResponse{ - Host: host, - Checks: &api.HostChecks{ - Gouging: hi.UnusableResult.gougingBreakdown.Gouging(), - GougingBreakdown: hi.UnusableResult.gougingBreakdown, - Score: hi.UnusableResult.scoreBreakdown.Score(), - ScoreBreakdown: hi.UnusableResult.scoreBreakdown, - Usable: hi.Usable, - UnusableReasons: hi.UnusableResult.reasons(), - }, - }) - if wanted > 0 && len(hostInfos) == wanted { - return hostInfos, nil // we're done. - } - keptHosts++ - } - - // if no hosts were kept from this batch, double the limit. - if limit > 0 && keptHosts == 0 { - limit *= 2 - } - } -} - -func isValidUsabilityFilterMode(usabilityMode string) bool { - switch usabilityMode { - case api.UsabilityFilterModeUsable: - case api.UsabilityFilterModeUnusable: - case api.UsabilityFilterModeAll: - case "": - default: - return false - } - return true -} diff --git a/stores/hostdb.go b/stores/hostdb.go index 8975c7006..a8763f2ca 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -651,6 +651,7 @@ func (ss *SQLStore) HostInfos(ctx context.Context, autopilotID string, filterMod // fetch host info var infos []dbHostInfo if err := query. + Debug(). Offset(offset). Limit(limit). Find(&infos). diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index 750ae65f5..8b154abc2 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -1230,6 +1230,7 @@ func TestHostInfo(t *testing.T) { } else if len(his) != 0 { t.Fatal("unexpected") } + // update info want.Usability.Blocked = false want.Usability.Offline = false From 50b7d76f5ad01ead2668e62ef06a7a488bdeef89 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 19 Mar 2024 09:24:04 +0100 Subject: [PATCH 05/23] stores: remove usability unknonwn --- api/host.go | 12 +++--------- autopilot/autopilot.go | 3 --- autopilot/contractor.go | 4 ++-- autopilot/hostfilter.go | 7 +------ autopilot/hostscore_test.go | 16 ++++++++-------- internal/test/e2e/cluster_test.go | 4 ++-- stores/hostdb.go | 11 ++++------- stores/hostdb_test.go | 2 -- .../mysql/main/migration_00007_host_info.sql | 2 -- stores/migrations/mysql/main/schema.sql | 2 -- .../sqlite/main/migration_00007_host_info.sql | 2 -- stores/migrations/sqlite/main/schema.sql | 3 +-- 12 files changed, 21 insertions(+), 47 deletions(-) diff --git a/api/host.go b/api/host.go index 11767979c..2b198474b 100644 --- a/api/host.go +++ b/api/host.go @@ -40,7 +40,6 @@ var ( ErrUsabilityHostNotAcceptingContracts = errors.New("host is not accepting contracts") ErrUsabilityHostNotCompletingScan = errors.New("host is not completing scan") ErrUsabilityHostNotAnnounced = errors.New("host is not announced") - ErrUsabilityUnknown = errors.New("unknown") ) type ( @@ -196,7 +195,6 @@ type ( NotAcceptingContracts bool `json:"notAcceptingContracts"` NotAnnounced bool `json:"notAnnounced"` NotCompletingScan bool `json:"notCompletingScan"` - Unknown bool `json:"unknown"` } ) @@ -235,7 +233,7 @@ func (hgb HostGougingBreakdown) String() string { return strings.Join(reasons, ";") } -func (sb HostScoreBreakdown) Score() float64 { +func (sb HostScoreBreakdown) TotalScore() float64 { return sb.Age * sb.Collateral * sb.Interactions * sb.StorageRemaining * sb.Uptime * sb.Version * sb.Prices } @@ -247,8 +245,7 @@ func (ub HostUsabilityBreakdown) Usable() bool { !ub.Gouging && !ub.NotAcceptingContracts && !ub.NotAnnounced && - !ub.NotCompletingScan && - !ub.Unknown + !ub.NotCompletingScan } func (ub HostUsabilityBreakdown) UnusableReasons() []string { @@ -277,9 +274,6 @@ func (ub HostUsabilityBreakdown) UnusableReasons() []string { if ub.NotCompletingScan { reasons = append(reasons, ErrUsabilityHostNotCompletingScan.Error()) } - if ub.Unknown { - reasons = append(reasons, ErrUsabilityUnknown.Error()) - } return reasons } @@ -289,7 +283,7 @@ func (hi HostInfo) ToHostInfoReponse() HostInfoResponse { Checks: &HostChecks{ Gouging: hi.Usability.Gouging, GougingBreakdown: hi.Gouging, - Score: hi.Score.Score(), + Score: hi.Score.TotalScore(), ScoreBreakdown: hi.Score, Usable: hi.Usability.Usable(), UnusableReasons: hi.Usability.UnusableReasons(), diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index c8df13279..beb32e589 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -809,9 +809,6 @@ func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Cu if hi.Usability.NotCompletingScan { resp.Unusable.NotScanned++ } - if hi.Usability.Unknown { - resp.Unusable.Unknown++ - } if hi.Gouging.ContractErr != "" { resp.Unusable.Gouging.Contract++ } diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 814cba2d0..a4504ab29 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -1357,13 +1357,13 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.Host, us h.PriceTable.HostBlockHeight = cs.BlockHeight hi := calculateHostInfo(state.cfg, state.rs, gc, h, minScore, storedData[h.PublicKey]) if hi.Usability.Usable() { - candidates = append(candidates, scoredHost{h, hi.Score.Score()}) + candidates = append(candidates, scoredHost{h, hi.Score.TotalScore()}) continue } // keep track of unusable host results unusableHosts.track(hi.Usability) - if hi.Score.Score() == 0 { + if hi.Score.TotalScore() == 0 { zeros++ } unusable++ diff --git a/autopilot/hostfilter.go b/autopilot/hostfilter.go index 462be4588..c1a55380d 100644 --- a/autopilot/hostfilter.go +++ b/autopilot/hostfilter.go @@ -49,7 +49,6 @@ type unusableHostsBreakdown struct { notacceptingcontracts uint64 notannounced uint64 notcompletingscan uint64 - unknown uint64 } func (u *unusableHostsBreakdown) track(ub api.HostUsabilityBreakdown) { @@ -77,9 +76,6 @@ func (u *unusableHostsBreakdown) track(ub api.HostUsabilityBreakdown) { if ub.NotCompletingScan { u.notcompletingscan++ } - if ub.Unknown { - u.unknown++ - } } func (u *unusableHostsBreakdown) keysAndValues() []interface{} { @@ -92,7 +88,6 @@ func (u *unusableHostsBreakdown) keysAndValues() []interface{} { "notacceptingcontracts", u.notacceptingcontracts, "notcompletingscan", u.notcompletingscan, "notannounced", u.notannounced, - "unknown", u.unknown, } for i := 0; i < len(values); i += 2 { if values[i+1].(uint64) == 0 { @@ -143,7 +138,7 @@ func calculateHostInfo(cfg api.AutopilotConfig, rs api.RedundancySettings, gc wo // checks in its cost calculations needed to calculate the period // cost sb = hostScore(cfg, h, storedData, rs.Redundancy()) - if sb.Score() < minScore { + if sb.TotalScore() < minScore { ub.LowScore = true } } diff --git a/autopilot/hostscore_test.go b/autopilot/hostscore_test.go index e48417235..464369052 100644 --- a/autopilot/hostscore_test.go +++ b/autopilot/hostscore_test.go @@ -48,7 +48,7 @@ func TestHostScore(t *testing.T) { // assert age affects the score h1.KnownSince = time.Now().Add(-1 * day) - if hostScore(cfg, h1, 0, redundancy).Score() <= hostScore(cfg, h2, 0, redundancy).Score() { + if hostScore(cfg, h1, 0, redundancy).TotalScore() <= hostScore(cfg, h2, 0, redundancy).TotalScore() { t.Fatal("unexpected") } @@ -57,21 +57,21 @@ func TestHostScore(t *testing.T) { settings.Collateral = settings.Collateral.Div64(2) settings.MaxCollateral = settings.MaxCollateral.Div64(2) h1 = newHost(settings) // reset - if hostScore(cfg, h1, 0, redundancy).Score() <= hostScore(cfg, h2, 0, redundancy).Score() { + if hostScore(cfg, h1, 0, redundancy).TotalScore() <= hostScore(cfg, h2, 0, redundancy).TotalScore() { t.Fatal("unexpected") } // assert interactions affect the score h1 = newHost(newTestHostSettings()) // reset h1.Interactions.SuccessfulInteractions++ - if hostScore(cfg, h1, 0, redundancy).Score() <= hostScore(cfg, h2, 0, redundancy).Score() { + if hostScore(cfg, h1, 0, redundancy).TotalScore() <= hostScore(cfg, h2, 0, redundancy).TotalScore() { t.Fatal("unexpected") } // assert uptime affects the score h2 = newHost(newTestHostSettings()) // reset h2.Interactions.SecondToLastScanSuccess = false - if hostScore(cfg, h1, 0, redundancy).Score() <= hostScore(cfg, h2, 0, redundancy).Score() || ageScore(h1) != ageScore(h2) { + if hostScore(cfg, h1, 0, redundancy).TotalScore() <= hostScore(cfg, h2, 0, redundancy).TotalScore() || ageScore(h1) != ageScore(h2) { t.Fatal("unexpected") } @@ -79,28 +79,28 @@ func TestHostScore(t *testing.T) { h2Settings := newTestHostSettings() h2Settings.Version = "1.5.6" // lower h2 = newHost(h2Settings) // reset - if hostScore(cfg, h1, 0, redundancy).Score() <= hostScore(cfg, h2, 0, redundancy).Score() { + if hostScore(cfg, h1, 0, redundancy).TotalScore() <= hostScore(cfg, h2, 0, redundancy).TotalScore() { t.Fatal("unexpected") } // asseret remaining storage affects the score. h1 = newHost(newTestHostSettings()) // reset h2.Settings.RemainingStorage = 100 - if hostScore(cfg, h1, 0, redundancy).Score() <= hostScore(cfg, h2, 0, redundancy).Score() { + if hostScore(cfg, h1, 0, redundancy).TotalScore() <= hostScore(cfg, h2, 0, redundancy).TotalScore() { t.Fatal("unexpected") } // assert MaxCollateral affects the score. h2 = newHost(newTestHostSettings()) // reset h2.PriceTable.MaxCollateral = types.ZeroCurrency - if hostScore(cfg, h1, 0, redundancy).Score() <= hostScore(cfg, h2, 0, redundancy).Score() { + if hostScore(cfg, h1, 0, redundancy).TotalScore() <= hostScore(cfg, h2, 0, redundancy).TotalScore() { t.Fatal("unexpected") } // assert price affects the score. h2 = newHost(newTestHostSettings()) // reset h2.PriceTable.WriteBaseCost = types.Siacoins(1) - if hostScore(cfg, h1, 0, redundancy).Score() <= hostScore(cfg, h2, 0, redundancy).Score() { + if hostScore(cfg, h1, 0, redundancy).TotalScore() <= hostScore(cfg, h2, 0, redundancy).TotalScore() { t.Fatal("unexpected") } } diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 2346f7019..18296a1f8 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -153,7 +153,7 @@ func TestNewTestCluster(t *testing.T) { if err != nil { t.Fatal(err) } - if hi.Checks.ScoreBreakdown.Score() == 0 { + if hi.Checks.ScoreBreakdown.TotalScore() == 0 { js, _ := json.MarshalIndent(hi.Checks.ScoreBreakdown, "", " ") t.Fatalf("score shouldn't be 0 because that means one of the fields was 0: %s", string(js)) } @@ -175,7 +175,7 @@ func TestNewTestCluster(t *testing.T) { allHosts := make(map[types.PublicKey]struct{}) for _, hi := range hostInfos { - if hi.Checks.ScoreBreakdown.Score() == 0 { + if hi.Checks.ScoreBreakdown.TotalScore() == 0 { js, _ := json.MarshalIndent(hi.Checks.ScoreBreakdown, "", " ") t.Fatalf("score shouldn't be 0 because that means one of the fields was 0: %s", string(js)) } diff --git a/stores/hostdb.go b/stores/hostdb.go index a8763f2ca..1d0b1959d 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -100,7 +100,6 @@ type ( UsabilityNotAcceptingContracts bool `gorm:"index:idx_host_infos_usability_not_accepting_contracts"` UsabilityNotAnnounced bool `gorm:"index:idx_host_infos_usability_not_announced"` UsabilityNotCompletingScan bool `gorm:"index:idx_host_infos_usability_not_completing_scan"` - UsabilityUnknown bool `gorm:"index:idx_host_infos_usability_unknown"` // score ScoreAge float64 `gorm:"index:idx_host_infos_score_age"` @@ -370,7 +369,6 @@ func (hi dbHostInfo) convert() api.HostInfo { NotAcceptingContracts: hi.UsabilityNotAcceptingContracts, NotAnnounced: hi.UsabilityNotAnnounced, NotCompletingScan: hi.UsabilityNotCompletingScan, - Unknown: hi.UsabilityUnknown, }, } } @@ -388,7 +386,6 @@ func convertHostInfo(apID, hID uint, gouging api.HostGougingBreakdown, score api UsabilityNotAcceptingContracts: usability.NotAcceptingContracts, UsabilityNotAnnounced: usability.NotAnnounced, UsabilityNotCompletingScan: usability.NotCompletingScan, - UsabilityUnknown: usability.Unknown, ScoreAge: score.Age, ScoreCollateral: score.Collateral, @@ -619,11 +616,11 @@ func (ss *SQLStore) HostInfos(ctx context.Context, autopilotID string, filterMod // apply usability filter switch usabilityMode { case api.UsabilityFilterModeUsable: - query = query.Where("usability_blocked = ? AND usability_offline = ? AND usability_low_score = ? AND usability_redundant_ip = ? AND usability_gouging = ? AND usability_not_accepting_contracts = ? AND usability_not_announced = ? AND usability_not_completing_scan = ? AND usability_unknown = ?", - false, false, false, false, false, false, false, false, false) + query = query.Where("usability_blocked = ? AND usability_offline = ? AND usability_low_score = ? AND usability_redundant_ip = ? AND usability_gouging = ? AND usability_not_accepting_contracts = ? AND usability_not_announced = ? AND usability_not_completing_scan = ?", + false, false, false, false, false, false, false, false) case api.UsabilityFilterModeUnusable: - query = query.Where("usability_blocked = ? OR usability_offline = ? OR usability_low_score = ? OR usability_redundant_ip = ? OR usability_gouging = ? OR usability_not_accepting_contracts = ? OR usability_not_announced = ? OR usability_not_completing_scan = ? OR usability_unknown = ?", - true, true, true, true, true, true, true, true, true) + query = query.Where("usability_blocked = ? OR usability_offline = ? OR usability_low_score = ? OR usability_redundant_ip = ? OR usability_gouging = ? OR usability_not_accepting_contracts = ? OR usability_not_announced = ? OR usability_not_completing_scan = ?", + true, true, true, true, true, true, true, true) case api.UsabilityFilterModeAll: // nothing to do default: diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index 8b154abc2..bf22fb20b 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -1240,7 +1240,6 @@ func TestHostInfo(t *testing.T) { want.Usability.NotAcceptingContracts = false want.Usability.NotAnnounced = false want.Usability.NotCompletingScan = false - want.Usability.Unknown = false err = ss.UpdateHostInfo(context.Background(), "foo", types.PublicKey{1}, want.Gouging, want.Score, want.Usability) if err != nil { t.Fatal(err) @@ -1376,7 +1375,6 @@ func newTestHostInfo(h hostdb.Host) api.HostInfo { NotAcceptingContracts: true, NotAnnounced: true, NotCompletingScan: true, - Unknown: true, }, } } diff --git a/stores/migrations/mysql/main/migration_00007_host_info.sql b/stores/migrations/mysql/main/migration_00007_host_info.sql index 69864b3e1..c13f5c396 100644 --- a/stores/migrations/mysql/main/migration_00007_host_info.sql +++ b/stores/migrations/mysql/main/migration_00007_host_info.sql @@ -14,7 +14,6 @@ CREATE TABLE `host_infos` ( `usability_not_accepting_contracts` boolean NOT NULL DEFAULT false, `usability_not_announced` boolean NOT NULL DEFAULT false, `usability_not_completing_scan` boolean NOT NULL DEFAULT false, - `usability_unknown` boolean NOT NULL DEFAULT false, `score_age` double NOT NULL, `score_collateral` double NOT NULL, @@ -40,7 +39,6 @@ CREATE TABLE `host_infos` ( INDEX `idx_host_infos_usability_not_accepting_contracts` (`usability_not_accepting_contracts`), INDEX `idx_host_infos_usability_not_announced` (`usability_not_announced`), INDEX `idx_host_infos_usability_not_completing_scan` (`usability_not_completing_scan`), - INDEX `idx_host_infos_usability_unknown` (`usability_unknown`), INDEX `idx_host_infos_score_age` (`score_age`), INDEX `idx_host_infos_score_collateral` (`score_collateral`), INDEX `idx_host_infos_score_interactions` (`score_interactions`), diff --git a/stores/migrations/mysql/main/schema.sql b/stores/migrations/mysql/main/schema.sql index 4eaa91499..e39b7f963 100644 --- a/stores/migrations/mysql/main/schema.sql +++ b/stores/migrations/mysql/main/schema.sql @@ -438,7 +438,6 @@ CREATE TABLE `host_infos` ( `usability_not_accepting_contracts` boolean NOT NULL DEFAULT false, `usability_not_announced` boolean NOT NULL DEFAULT false, `usability_not_completing_scan` boolean NOT NULL DEFAULT false, - `usability_unknown` boolean NOT NULL DEFAULT false, `score_age` double NOT NULL, `score_collateral` double NOT NULL, @@ -464,7 +463,6 @@ CREATE TABLE `host_infos` ( INDEX `idx_host_infos_usability_not_accepting_contracts` (`usability_not_accepting_contracts`), INDEX `idx_host_infos_usability_not_announced` (`usability_not_announced`), INDEX `idx_host_infos_usability_not_completing_scan` (`usability_not_completing_scan`), - INDEX `idx_host_infos_usability_unknown` (`usability_unknown`), INDEX `idx_host_infos_score_age` (`score_age`), INDEX `idx_host_infos_score_collateral` (`score_collateral`), INDEX `idx_host_infos_score_interactions` (`score_interactions`), diff --git a/stores/migrations/sqlite/main/migration_00007_host_info.sql b/stores/migrations/sqlite/main/migration_00007_host_info.sql index 5d425dfe7..910dd637c 100644 --- a/stores/migrations/sqlite/main/migration_00007_host_info.sql +++ b/stores/migrations/sqlite/main/migration_00007_host_info.sql @@ -14,7 +14,6 @@ CREATE TABLE `host_infos` ( `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, `usability_not_announced` INTEGER NOT NULL DEFAULT 0, `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, - `usability_unknown` INTEGER NOT NULL DEFAULT 0, `score_age` REAL NOT NULL, `score_collateral` REAL NOT NULL, @@ -44,7 +43,6 @@ CREATE INDEX `idx_host_infos_usability_gouging` ON `host_infos` (`usability_goug CREATE INDEX `idx_host_infos_usability_not_accepting_contracts` ON `host_infos` (`usability_not_accepting_contracts`); CREATE INDEX `idx_host_infos_usability_not_announced` ON `host_infos` (`usability_not_announced`); CREATE INDEX `idx_host_infos_usability_not_completing_scan` ON `host_infos` (`usability_not_completing_scan`); -CREATE INDEX `idx_host_infos_usability_unknown` ON `host_infos` (`usability_unknown`); CREATE INDEX `idx_host_infos_score_age` ON `host_infos` (`score_age`); CREATE INDEX `idx_host_infos_score_collateral` ON `host_infos` (`score_collateral`); CREATE INDEX `idx_host_infos_score_interactions` ON `host_infos` (`score_interactions`); diff --git a/stores/migrations/sqlite/main/schema.sql b/stores/migrations/sqlite/main/schema.sql index 5ec7a2b0e..791fce1ca 100644 --- a/stores/migrations/sqlite/main/schema.sql +++ b/stores/migrations/sqlite/main/schema.sql @@ -150,7 +150,7 @@ CREATE TABLE `object_user_metadata` (`id` integer PRIMARY KEY AUTOINCREMENT,`cre CREATE UNIQUE INDEX `idx_object_user_metadata_key` ON `object_user_metadata`(`db_object_id`,`db_multipart_upload_id`,`key`); -- dbHostInfo -CREATE TABLE `host_infos` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `created_at` datetime, `db_autopilot_id` INTEGER NOT NULL, `db_host_id` INTEGER NOT NULL, `usability_blocked` INTEGER NOT NULL DEFAULT 0, `usability_offline` INTEGER NOT NULL DEFAULT 0, `usability_low_score` INTEGER NOT NULL DEFAULT 0, `usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, `usability_gouging` INTEGER NOT NULL DEFAULT 0, `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, `usability_not_announced` INTEGER NOT NULL DEFAULT 0, `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, `usability_unknown` INTEGER NOT NULL DEFAULT 0, `score_age` REAL NOT NULL, `score_collateral` REAL NOT NULL, `score_interactions` REAL NOT NULL, `score_storage_remaining` REAL NOT NULL, `score_uptime` REAL NOT NULL, `score_version` REAL NOT NULL, `score_prices` REAL NOT NULL, `gouging_contract_err` TEXT, `gouging_download_err` TEXT, `gouging_gouging_err` TEXT, `gouging_prune_err` TEXT, `gouging_upload_err` TEXT, FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE); +CREATE TABLE `host_infos` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `created_at` datetime, `db_autopilot_id` INTEGER NOT NULL, `db_host_id` INTEGER NOT NULL, `usability_blocked` INTEGER NOT NULL DEFAULT 0, `usability_offline` INTEGER NOT NULL DEFAULT 0, `usability_low_score` INTEGER NOT NULL DEFAULT 0, `usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, `usability_gouging` INTEGER NOT NULL DEFAULT 0, `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, `usability_not_announced` INTEGER NOT NULL DEFAULT 0, `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, `score_age` REAL NOT NULL, `score_collateral` REAL NOT NULL, `score_interactions` REAL NOT NULL, `score_storage_remaining` REAL NOT NULL, `score_uptime` REAL NOT NULL, `score_version` REAL NOT NULL, `score_prices` REAL NOT NULL, `gouging_contract_err` TEXT, `gouging_download_err` TEXT, `gouging_gouging_err` TEXT, `gouging_prune_err` TEXT, `gouging_upload_err` TEXT, FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE); CREATE UNIQUE INDEX `idx_host_infos_id` ON `host_infos` (`db_autopilot_id`, `db_host_id`); CREATE INDEX `idx_host_infos_usability_blocked` ON `host_infos` (`usability_blocked`); CREATE INDEX `idx_host_infos_usability_offline` ON `host_infos` (`usability_offline`); @@ -160,7 +160,6 @@ CREATE INDEX `idx_host_infos_usability_gouging` ON `host_infos` (`usability_goug CREATE INDEX `idx_host_infos_usability_not_accepting_contracts` ON `host_infos` (`usability_not_accepting_contracts`); CREATE INDEX `idx_host_infos_usability_not_announced` ON `host_infos` (`usability_not_announced`); CREATE INDEX `idx_host_infos_usability_not_completing_scan` ON `host_infos` (`usability_not_completing_scan`); -CREATE INDEX `idx_host_infos_usability_unknown` ON `host_infos` (`usability_unknown`); CREATE INDEX `idx_host_infos_score_age` ON `host_infos` (`score_age`); CREATE INDEX `idx_host_infos_score_collateral` ON `host_infos` (`score_collateral`); CREATE INDEX `idx_host_infos_score_interactions` ON `host_infos` (`score_interactions`); From 37c725e3d84f2ad1015dcecffe4263348ee54be9 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 19 Mar 2024 09:39:29 +0100 Subject: [PATCH 06/23] api: fix japecheck errors --- autopilot/client.go | 2 +- bus/client/hosts.go | 2 +- stores/hostdb_test.go | 7 +------ 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/autopilot/client.go b/autopilot/client.go index 41b0b3207..4d491f457 100644 --- a/autopilot/client.go +++ b/autopilot/client.go @@ -41,7 +41,7 @@ func (c *Client) HostInfo(hostKey types.PublicKey) (resp api.HostInfoResponse, e // HostInfo returns information about all hosts. func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode string, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostInfoResponse, err error) { - err = c.c.POST("/hosts", api.SearchHostsRequest{ + err = c.c.POST("/hosts", api.HostInfosRequest{ Offset: offset, Limit: limit, FilterMode: filterMode, diff --git a/bus/client/hosts.go b/bus/client/hosts.go index beda5950b..858989ec7 100644 --- a/bus/client/hosts.go +++ b/bus/client/hosts.go @@ -119,7 +119,7 @@ func (c *Client) UpdateHostInfo(ctx context.Context, autopilotID string, hostKey // HostInfos returns the host info for all hosts known to the autopilot with the given identifier. func (c *Client) HostInfos(ctx context.Context, autopilotID string, opts api.HostInfoOptions) (hostInfos []api.HostInfo, err error) { - err = c.c.WithContext(ctx).POST(fmt.Sprintf("/autopilot/%s", autopilotID), api.HostInfosRequest{ + err = c.c.WithContext(ctx).POST(fmt.Sprintf("/autopilot/%s/hosts", autopilotID), api.HostInfosRequest{ Offset: opts.Offset, Limit: opts.Limit, FilterMode: opts.FilterMode, diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index bf22fb20b..8a75caf6d 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -5,7 +5,6 @@ import ( "context" "errors" "fmt" - "os" "reflect" "testing" "time" @@ -1066,11 +1065,7 @@ func TestAnnouncementMaxAge(t *testing.T) { } func TestHostInfo(t *testing.T) { - cfg := defaultTestSQLStoreConfig - cfg.persistent = true - cfg.dir = "/Users/peterjan/testing" - os.RemoveAll(cfg.dir) - ss := newTestSQLStore(t, cfg) + ss := newTestSQLStore(t, defaultTestSQLStoreConfig) defer ss.Close() // fetch info for a non-existing autopilot From c011dc835a4899cc3df38f72173d9de81944ef69 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 19 Mar 2024 11:26:09 +0100 Subject: [PATCH 07/23] autopilot: update host filter --- autopilot/hostfilter.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/autopilot/hostfilter.go b/autopilot/hostfilter.go index 33a6daa85..7113fe5bc 100644 --- a/autopilot/hostfilter.go +++ b/autopilot/hostfilter.go @@ -106,15 +106,18 @@ func calculateHostInfo(cfg api.AutopilotConfig, rs api.RedundancySettings, gc wo } // prepare host breakdown fields - var ub api.HostUsabilityBreakdown var gb api.HostGougingBreakdown var sb api.HostScoreBreakdown + var ub api.HostUsabilityBreakdown + + // blocked status does not influence what host info is calculated + if h.Blocked { + ub.Blocked = true + } - // populate host info fields + // calculate remaining host info fields if !h.IsAnnounced() { ub.NotAnnounced = true - } else if h.Blocked { - ub.Blocked = true } else if !h.Scanned { ub.NotCompletingScan = true } else { From 61c7c971a28bd50adb1a8cee847728b57df3e6a5 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 19 Mar 2024 11:34:23 +0100 Subject: [PATCH 08/23] contractor: remove cached fields --- autopilot/contractor.go | 76 ++++++----------------------------------- 1 file changed, 11 insertions(+), 65 deletions(-) diff --git a/autopilot/contractor.go b/autopilot/contractor.go index ef4ff6e28..ba6c6a5d9 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -100,9 +100,6 @@ type ( pruning bool pruningLastStart time.Time - - cachedDataStored map[types.PublicKey]uint64 - cachedMinScore float64 } scoredHost struct { @@ -275,20 +272,14 @@ func (c *contractor) performContractMaintenance(ctx context.Context, w Worker) ( c.logger.Warn("could not calculate min score, no hosts found") } - // update cache. - c.mu.Lock() - c.cachedDataStored = hostData - c.cachedMinScore = minScore - c.mu.Unlock() - // run host checks - err = c.runHostChecks(ctx, hosts) + err = c.runHostChecks(ctx, hosts, hostData, minScore) if err != nil { return false, fmt.Errorf("failed to run host checks, err: %v", err) } // run contract checks - updatedSet, toArchive, toStopUsing, toRefresh, toRenew, err := c.runContractChecks(ctx, w, contracts, isInCurrentSet, minScore) + updatedSet, toArchive, toStopUsing, toRefresh, toRenew, err := c.runContractChecks(ctx, contracts, isInCurrentSet) if err != nil { return false, fmt.Errorf("failed to run contract checks, err: %v", err) } @@ -637,7 +628,7 @@ func (c *contractor) performWalletMaintenance(ctx context.Context) error { return nil } -func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts []api.Contract, inCurrentSet map[types.FileContractID]struct{}, minScore float64) (toKeep []api.ContractMetadata, toArchive, toStopUsing map[types.FileContractID]string, toRefresh, toRenew []contractInfo, _ error) { +func (c *contractor) runContractChecks(ctx context.Context, contracts []api.Contract, inCurrentSet map[types.FileContractID]struct{}) (toKeep []api.ContractMetadata, toArchive, toStopUsing map[types.FileContractID]string, toRefresh, toRenew []contractInfo, _ error) { if c.ap.isStopped() { return } @@ -695,6 +686,7 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts // convenience variables fcid := contract.ID + hk := contract.HostKey // check if contract is ready to be archived. if cs.BlockHeight > contract.EndHeight()-c.revisionSubmissionBuffer { @@ -712,8 +704,7 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts } // fetch host from hostdb - hk := contract.HostKey - host, err := c.ap.bus.Host(ctx, hk) + host, err := c.ap.bus.HostInfo(ctx, c.ap.id, hk) if err != nil { c.logger.Errorw(fmt.Sprintf("missing host, err: %v", err), "hk", hk) toStopUsing[fcid] = api.ErrUsabilityHostNotFound.Error() @@ -722,42 +713,15 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts } // if the host is blocked we ignore it, it might be unblocked later - if host.Blocked { + if host.Usability.Blocked { c.logger.Infow("unusable host", "hk", hk, "fcid", fcid, "reasons", api.ErrUsabilityHostBlocked.Error()) toStopUsing[fcid] = api.ErrUsabilityHostBlocked.Error() continue } - // if the host doesn't have a valid pricetable, update it if we were - // able to obtain a revision - invalidPT := contract.Revision == nil - if contract.Revision != nil { - if err := refreshPriceTable(ctx, w, &host.Host); err != nil { - c.logger.Errorf("could not fetch price table for host %v: %v", host.PublicKey, err) - invalidPT = true - } - } - - // refresh the consensus state - if css, err := c.ap.bus.ConsensusState(ctx); err != nil { - c.logger.Errorf("could not fetch consensus state, err: %v", err) - } else { - cs = css - } - - // use a new gouging checker for every contract - gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) - - // set the host's block height to ours to disable the height check in - // the gouging checks, in certain edge cases the renter might unsync and - // would therefor label all hosts as unusable and go on to create a - // whole new set of contracts with new hosts - host.PriceTable.HostBlockHeight = cs.BlockHeight - - // decide whether the host is still good - hi := calculateHostInfo(state.cfg, state.rs, gc, host, minScore, contract.FileSize()) - if !hi.Usability.Usable() { - reasons := hi.Usability.UnusableReasons() + // check if the host is still usable + if !host.Usability.Usable() { + reasons := host.Usability.UnusableReasons() toStopUsing[fcid] = strings.Join(reasons, ",") c.logger.Infow("unusable host", "hk", hk, "fcid", fcid, "reasons", reasons) continue @@ -778,20 +742,8 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts continue // can't perform contract checks without revision } - // if we were not able to get a valid price table for the host, but we - // did pass the host checks, we only want to be lenient if this contract - // is in the current set and only for a certain number of times, - // controlled by maxKeepLeeway - if invalidPT { - if _, found := inCurrentSet[fcid]; !found || remainingKeepLeeway == 0 { - toStopUsing[fcid] = "no valid price table" - continue - } - remainingKeepLeeway-- // we let it slide - } - // decide whether the contract is still good - ci := contractInfo{contract: contract, priceTable: host.PriceTable.HostPriceTable, settings: host.Settings} + ci := contractInfo{contract: contract, priceTable: host.Host.PriceTable.HostPriceTable, settings: host.Host.Settings} usable, recoverable, refresh, renew, reasons := c.isUsableContract(state.cfg, state, ci, cs.BlockHeight, ipFilter) ci.usable = usable ci.recoverable = recoverable @@ -822,7 +774,7 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts return toKeep, toArchive, toStopUsing, toRefresh, toRenew, nil } -func (c *contractor) runHostChecks(ctx context.Context, hosts []hostdb.HostInfo) error { +func (c *contractor) runHostChecks(ctx context.Context, hosts []hostdb.HostInfo, hostData map[types.PublicKey]uint64, minScore float64) error { // convenience variables state := c.ap.State() @@ -835,12 +787,6 @@ func (c *contractor) runHostChecks(ctx context.Context, hosts []hostdb.HostInfo) // create gouging checker gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) - // grab min score and host data from cache - c.mu.Lock() - minScore := c.cachedMinScore - hostData := c.cachedDataStored - c.mu.Unlock() - // update host info for _, h := range hosts { h.PriceTable.HostBlockHeight = cs.BlockHeight // ignore HostBlockHeight From 06070ac03ab1e9ad529ecba101672fb3346f82f7 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 19 Mar 2024 11:37:21 +0100 Subject: [PATCH 09/23] autopilot: remove TODO --- autopilot/autopilot.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 80c50e1a1..a39818c26 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -696,15 +696,7 @@ func (ap *Autopilot) hostHandlerGET(jc jape.Context) { } hi, err := ap.bus.HostInfo(jc.Request.Context(), ap.id, hk) - if utils.IsErr(err, api.ErrHostInfoNotFound) { - // TODO PJ: we used to calculate the host info here on the fly, maybe we - // should keep doing that but maybe we can get away with this too... - jc.Encode(api.HostInfoResponse{ - Host: h.Host, - Checks: nil, - }) - return - } else if jc.Check("failed to get host info", err) != nil { + if jc.Check("failed to get host info", err) != nil { return } From 0d438a7184e9a121e6b9f738b1862465dd3dc969 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 19 Mar 2024 11:41:54 +0100 Subject: [PATCH 10/23] lint: fix --- autopilot/autopilot.go | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index a39818c26..9b82e20ee 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -690,8 +690,7 @@ func (ap *Autopilot) hostHandlerGET(jc jape.Context) { } // TODO: remove on next major release - h, err := compatV105HostInfo(jc.Request.Context(), ap.State(), ap.bus, hk) - if jc.Check("failed to get host info", err) != nil { + if jc.Check("failed to get host info", compatV105HostInfo(jc.Request.Context(), ap.State(), ap.bus, hk)) != nil { return } @@ -937,42 +936,42 @@ func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg // compatV105HostInfo performs some state checks and bus calls we no longer // need, but are necessary checks to make sure our API is consistent. This // should be considered for removal when releasing a new major version. -func compatV105HostInfo(ctx context.Context, s state, b Bus, hk types.PublicKey) (*hostdb.HostInfo, error) { +func compatV105HostInfo(ctx context.Context, s state, b Bus, hk types.PublicKey) error { // state checks if s.cfg.Contracts.Allowance.IsZero() { - return nil, fmt.Errorf("can not score hosts because contracts allowance is zero") + return fmt.Errorf("can not score hosts because contracts allowance is zero") } if s.cfg.Contracts.Amount == 0 { - return nil, fmt.Errorf("can not score hosts because contracts amount is zero") + return fmt.Errorf("can not score hosts because contracts amount is zero") } if s.cfg.Contracts.Period == 0 { - return nil, fmt.Errorf("can not score hosts because contract period is zero") + return fmt.Errorf("can not score hosts because contract period is zero") } // fetch host - host, err := b.Host(ctx, hk) + _, err := b.Host(ctx, hk) if err != nil { - return nil, fmt.Errorf("failed to fetch requested host from bus: %w", err) + return fmt.Errorf("failed to fetch requested host from bus: %w", err) } // other checks _, err = b.GougingSettings(ctx) if err != nil { - return nil, fmt.Errorf("failed to fetch gouging settings from bus: %w", err) + return fmt.Errorf("failed to fetch gouging settings from bus: %w", err) } _, err = b.RedundancySettings(ctx) if err != nil { - return nil, fmt.Errorf("failed to fetch redundancy settings from bus: %w", err) + return fmt.Errorf("failed to fetch redundancy settings from bus: %w", err) } _, err = b.ConsensusState(ctx) if err != nil { - return nil, fmt.Errorf("failed to fetch consensus state from bus: %w", err) + return fmt.Errorf("failed to fetch consensus state from bus: %w", err) } _, err = b.RecommendedFee(ctx) if err != nil { - return nil, fmt.Errorf("failed to fetch recommended fee from bus: %w", err) + return fmt.Errorf("failed to fetch recommended fee from bus: %w", err) } - return &host, nil + return nil } func compatV105UsabilityFilterModeCheck(usabilityMode string) error { From fb444908d6ded2003f3abbae38f48214e325ec35 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 19 Mar 2024 12:59:24 +0100 Subject: [PATCH 11/23] autopilot: update comment --- autopilot/autopilot.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 9b82e20ee..57876a6ad 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -933,9 +933,9 @@ func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg } } -// compatV105HostInfo performs some state checks and bus calls we no longer -// need, but are necessary checks to make sure our API is consistent. This -// should be considered for removal when releasing a new major version. +// compatV105HostInfo performs some state checks and bus calls we no longer need +// but are necessary checks to make sure our API is consistent. This should be +// removed in the next major release. func compatV105HostInfo(ctx context.Context, s state, b Bus, hk types.PublicKey) error { // state checks if s.cfg.Contracts.Allowance.IsZero() { From 864c08b7b12f5426b1775a3e7d0520deb3cec6dc Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 20 Mar 2024 15:02:36 +0100 Subject: [PATCH 12/23] contractor: fetch all hosts --- autopilot/contractor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 9c778004e..27cbd3834 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -240,7 +240,7 @@ func (c *contractor) performContractMaintenance(ctx context.Context, w Worker) ( } // fetch all hosts - hosts, err := c.ap.bus.SearchHosts(ctx, api.SearchHostOptions{Limit: -1, FilterMode: api.HostFilterModeAllowed}) + hosts, err := c.ap.bus.SearchHosts(ctx, api.SearchHostOptions{Limit: -1, FilterMode: api.HostFilterModeAll}) if err != nil { return false, err } From 535a4d9015dc9d096619682641720c4683e4a172 Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 25 Mar 2024 16:14:08 +0100 Subject: [PATCH 13/23] stores: add autopilot and usability filter to SearchHosts --- api/host.go | 12 ++----- autopilot/autopilot.go | 7 ++++ autopilot/contractor.go | 19 ++++++----- bus/bus.go | 74 +++-------------------------------------- bus/client/hosts.go | 2 ++ stores/hostdb.go | 42 +++++++++++++++++++++-- stores/hostdb_test.go | 71 ++++++++++++++++++++++++++++++++++----- 7 files changed, 131 insertions(+), 96 deletions(-) diff --git a/api/host.go b/api/host.go index 0691c4d56..46a5597b3 100644 --- a/api/host.go +++ b/api/host.go @@ -24,10 +24,6 @@ var ( // ErrHostNotFound is returned when a host can't be retrieved from the // database. ErrHostNotFound = errors.New("host doesn't exist in hostdb") - - // ErrHostInfoNotFound is returned when host info can't be retrieved from - // the database. - ErrHostInfoNotFound = errors.New("host info doesn't exist in hostdb") ) var ( @@ -71,11 +67,8 @@ type ( KeyIn []types.PublicKey `json:"keyIn"` } - // HostsRequest is the request type for the POST /autopilot/:id/hosts - // endpoint. - HostsRequest SearchHostsRequest - - // HostResponse is the response type for the /host/:hostkey endpoint. + // HostResponse is the response type for the GET + // /api/autopilot/host/:hostkey endpoint. HostResponse struct { Host hostdb.Host `json:"host"` Checks *HostChecks `json:"checks,omitempty"` @@ -120,6 +113,7 @@ type ( } SearchHostOptions struct { + AutopilotID string AddressContains string FilterMode string UsabilityMode string diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 9b635dcbd..97ea4e939 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -705,6 +705,13 @@ func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { var req api.SearchHostOptions if jc.Decode(&req) != nil { return + } else if req.AutopilotID != "" && req.AutopilotID != ap.id { + jc.Error(errors.New("invalid autopilot id"), http.StatusBadRequest) + return + } else { + // TODO: on next major release we should not re-use options between bus + // and autopilot API if we don't support all fields in both + req.AutopilotID = ap.id } // TODO: remove on next major release diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 270ea701f..0e85e4302 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -713,12 +713,22 @@ func (c *contractor) runContractChecks(ctx context.Context, contracts []api.Cont // fetch host from hostdb host, err := c.ap.bus.Host(ctx, hk) if err != nil { - c.logger.Errorw(fmt.Sprintf("missing host, err: %v", err), "hk", hk) + c.logger.Warn(fmt.Sprintf("missing host, err: %v", err), "hk", hk) toStopUsing[fcid] = api.ErrUsabilityHostNotFound.Error() notfound++ continue } + // fetch host checks + check, ok := host.Checks[c.ap.id] + if !ok { + // this is only possible due to developer error, if there is no + // check the host would have been missing, so we treat it the same + c.logger.Warnw("missing host check", "hk", hk) + toStopUsing[fcid] = api.ErrUsabilityHostNotFound.Error() + continue + } + // if the host is blocked we ignore it, it might be unblocked later if host.Blocked { c.logger.Infow("unusable host", "hk", hk, "fcid", fcid, "reasons", api.ErrUsabilityHostBlocked.Error()) @@ -726,13 +736,6 @@ func (c *contractor) runContractChecks(ctx context.Context, contracts []api.Cont continue } - // grab the host check - check, ok := host.Checks[c.ap.id] - if !ok { - c.logger.Errorw("missing host check", "hk", hk) - continue - } - // check if the host is still usable if !check.Usability.IsUsable() { reasons := check.Usability.UnusableReasons() diff --git a/bus/bus.go b/bus/bus.go index 1fc5cfff8..d9eba1e96 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -253,9 +253,7 @@ func (b *bus) Handler() http.Handler { "GET /autopilot/:id": b.autopilotsHandlerGET, "PUT /autopilot/:id": b.autopilotsHandlerPUT, - "GET /autopilot/:id/host/:hostkey": b.autopilotHostHandlerGET, - "PUT /autopilot/:id/host/:hostkey": b.autopilotHostHandlerPUT, - "POST /autopilot/:id/hosts": b.autopilotHostsHandlerGET, + "PUT /autopilot/:id/host/:hostkey/checks": b.autopilotHostChecksHandlerPUT, "GET /buckets": b.bucketsHandlerGET, "POST /buckets": b.bucketsHandlerPOST, @@ -780,8 +778,9 @@ func (b *bus) searchHostsHandlerPOST(jc jape.Context) { } // TODO: on the next major release: - // - properly default search params - // - properly validate and return 400 + // - properly default search params (currenlty no defaults are set) + // - properly validate and return 400 (currently validation is done in autopilot and the store) + hosts, err := b.hdb.SearchHosts(jc.Request.Context(), req.AutopilotID, req.FilterMode, req.UsabilityMode, req.AddressContains, req.KeyIn, req.Offset, req.Limit) if jc.Check(fmt.Sprintf("couldn't fetch hosts %d-%d", req.Offset, req.Offset+req.Limit), err) != nil { return @@ -1970,27 +1969,7 @@ func (b *bus) autopilotsHandlerPUT(jc jape.Context) { jc.Check("failed to update autopilot", b.as.UpdateAutopilot(jc.Request.Context(), ap)) } -func (b *bus) autopilotHostHandlerGET(jc jape.Context) { - var id string - if jc.DecodeParam("id", &id) != nil { - return - } - var hk types.PublicKey - if jc.DecodeParam("hostkey", &hk) != nil { - return - } - - hi, err := b.hdb.Host(jc.Request.Context(), hk) - if errors.Is(err, api.ErrAutopilotNotFound) { - jc.Error(err, http.StatusNotFound) - return - } else if jc.Check("failed to fetch host info", err) != nil { - return - } - jc.Encode(hi) -} - -func (b *bus) autopilotHostHandlerPUT(jc jape.Context) { +func (b *bus) autopilotHostChecksHandlerPUT(jc jape.Context) { var id string if jc.DecodeParam("id", &id) != nil { return @@ -2013,49 +1992,6 @@ func (b *bus) autopilotHostHandlerPUT(jc jape.Context) { } } -func (b *bus) autopilotHostsHandlerGET(jc jape.Context) { - var id string - if jc.DecodeParam("id", &id) != nil { - return - } - var req api.HostsRequest - if jc.Decode(&req) != nil { - return - } - - // validate filter mode - if fm := req.FilterMode; fm != "" { - if fm != api.HostFilterModeAll && - fm != api.HostFilterModeAllowed && - fm != api.HostFilterModeBlocked { - jc.Error(fmt.Errorf("invalid filter mode: '%v', allowed values are '%s', '%s', '%s'", fm, api.HostFilterModeAll, api.HostFilterModeAllowed, api.HostFilterModeBlocked), http.StatusBadRequest) - return - } - } - - // validate usability mode - if um := req.UsabilityMode; um != "" { - if um != api.UsabilityFilterModeUsable && - um != api.UsabilityFilterModeUnusable && - um != api.UsabilityFilterModeAll { - jc.Error(fmt.Errorf("invalid usability mode: '%v', allowed values are '%s', '%s', '%s'", um, api.UsabilityFilterModeAll, api.UsabilityFilterModeUsable, api.UsabilityFilterModeUnusable), http.StatusBadRequest) - return - } else if id == "" { - jc.Error(errors.New("usability mode requires autopilot id"), http.StatusBadRequest) - return - } - } - - his, err := b.hdb.SearchHosts(jc.Request.Context(), id, req.FilterMode, req.UsabilityMode, req.AddressContains, req.KeyIn, req.Offset, req.Limit) - if errors.Is(err, api.ErrAutopilotNotFound) { - jc.Error(err, http.StatusNotFound) - return - } else if jc.Check("failed to fetch host infos", err) != nil { - return - } - jc.Encode(his) -} - func (b *bus) contractTaxHandlerGET(jc jape.Context) { var payout types.Currency if jc.DecodeParam("payout", (*api.ParamCurrency)(&payout)) != nil { diff --git a/bus/client/hosts.go b/bus/client/hosts.go index e84e3cea9..4e1aeab30 100644 --- a/bus/client/hosts.go +++ b/bus/client/hosts.go @@ -80,9 +80,11 @@ func (c *Client) ResetLostSectors(ctx context.Context, hostKey types.PublicKey) // SearchHosts returns all hosts that match certain search criteria. func (c *Client) SearchHosts(ctx context.Context, opts api.SearchHostOptions) (hosts []api.Host, err error) { err = c.c.WithContext(ctx).POST("/search/hosts", api.SearchHostsRequest{ + AutopilotID: opts.AutopilotID, Offset: opts.Offset, Limit: opts.Limit, FilterMode: opts.FilterMode, + UsabilityMode: opts.UsabilityMode, AddressContains: opts.AddressContains, KeyIn: opts.KeyIn, }, &hosts) diff --git a/stores/hostdb.go b/stores/hostdb.go index 9c7be8e9e..a121ab7d0 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -627,9 +627,11 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, autopilotID, filterMode, us query := ss.db. Model(&dbHost{}). Scopes( + autopilotFilter(autopilotID), hostFilter(filterMode, ss.hasAllowlist(), ss.hasBlocklist()), hostNetAddress(addressContains), hostPublicKey(keyIn), + usabilityFilter(usabilityMode), ) // preload allowlist and blocklist @@ -639,12 +641,23 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, autopilotID, filterMode, us Preload("Blocklist") } - // preload host checks - query = query.Preload("Checks.DBAutopilot") + // filter checks + if autopilotID != "" { + query = query.Preload("Checks.DBAutopilot", "identifier = ?", autopilotID) + } else { + query = query.Preload("Checks.DBAutopilot") + } + // query = query. + // Preload("Checks.DBAutopilot"). + // Scopes( + // autopilotFilter(autopilotID), + // usabilityFilter(usabilityMode), + // ) var hosts []api.Host var fullHosts []dbHost err := query. + Debug(). Offset(offset). Limit(limit). FindInBatches(&fullHosts, hostRetrievalBatchSize, func(tx *gorm.DB, batch int) error { @@ -1089,6 +1102,17 @@ func hostPublicKey(keyIn []types.PublicKey) func(*gorm.DB) *gorm.DB { } } +// autopilotFilter can be used as a scope to filter host checks based on their +// autopilot +func autopilotFilter(autopilotID string) func(*gorm.DB) *gorm.DB { + return func(db *gorm.DB) *gorm.DB { + if autopilotID == "" { + return db.Preload("Checks.DBAutopilot") + } + return db.Preload("Checks.DBAutopilot", "identifier = ?", autopilotID) + } +} + // hostFilter can be used as a scope to filter hosts based on their filter mode, // returning either all, allowed or blocked hosts. func hostFilter(filterMode string, hasAllowlist, hasBlocklist bool) func(*gorm.DB) *gorm.DB { @@ -1120,6 +1144,20 @@ func hostFilter(filterMode string, hasAllowlist, hasBlocklist bool) func(*gorm.D } } +func usabilityFilter(usabilityMode string) func(*gorm.DB) *gorm.DB { + return func(db *gorm.DB) *gorm.DB { + switch usabilityMode { + case api.UsabilityFilterModeUsable: + db = db.Preload("Checks", "usability_blocked = ? AND usability_offline = ? AND usability_low_score = ? AND usability_redundant_ip = ? AND usability_gouging = ? AND usability_not_accepting_contracts = ? AND usability_not_announced = ? AND usability_not_completing_scan = ?", false, false, false, false, false, false, false, false) + case api.UsabilityFilterModeUnusable: + db = db.Preload("Checks", "usability_blocked = ? OR usability_offline = ? OR usability_low_score = ? OR usability_redundant_ip = ? OR usability_gouging = ? OR usability_not_accepting_contracts = ? OR usability_not_announced = ? OR usability_not_completing_scan = ?", true, true, true, true, true, true, true, true) + case api.UsabilityFilterModeAll: + // do nothing + } + return db + } +} + func (ss *SQLStore) isBlocked(h dbHost) (blocked bool) { ss.mu.Lock() defer ss.mu.Unlock() diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index 55a318e5c..196170b01 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -382,6 +382,61 @@ func TestSearchHosts(t *testing.T) { t.Fatal("unexpected", c3, ok) } + // assert autopilot filter is taken into account + his, err = ss.SearchHosts(context.Background(), ap1, api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if cnt != 3 { + t.Fatal("unexpected", cnt) + } + + // assert h1 and h2 have the expected checks + if c1, ok := his[0].Checks[ap1]; !ok || c1 != h1c { + t.Fatal("unexpected", c1, ok) + } else if c2, ok := his[1].Checks[ap1]; !ok || c2 != h2c1 { + t.Fatal("unexpected", c2, ok) + } else if _, ok := his[1].Checks[ap2]; ok { + t.Fatal("unexpected") + } + + // assert usability filter is taken into account + h2c1.Usability.RedundantIP = true + err = ss.UpdateHostCheck(context.Background(), ap1, hk2, h2c1) + if err != nil { + t.Fatal(err) + } + his, err = ss.SearchHosts(context.Background(), ap1, api.HostFilterModeAll, api.UsabilityFilterModeUsable, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if cnt != 3 { + t.Fatal("unexpected", cnt) + } + + // assert h1 and h2 have the expected checks + if c1, ok := his[0].Checks[ap1]; !ok || c1 != h1c { + t.Fatal("unexpected", c1, ok) + } else if _, ok := his[1].Checks[ap1]; ok { + t.Fatal("unexpected", ok) + } else if _, ok := his[1].Checks[ap2]; ok { + t.Fatal("unexpected") + } + + his, err = ss.SearchHosts(context.Background(), ap1, api.HostFilterModeAll, api.UsabilityFilterModeUnusable, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if cnt != 3 { + t.Fatal("unexpected", cnt) + } + + // assert h1 and h2 have the expected checks + if _, ok := his[0].Checks[ap1]; ok { + t.Fatal("unexpected") + } else if c2, ok := his[1].Checks[ap1]; !ok || c2 != h2c1 { + t.Fatal("unexpected", ok) + } else if _, ok := his[1].Checks[ap2]; ok { + t.Fatal("unexpected") + } + // assert cascade delete on host err = ss.db.Exec("DELETE FROM hosts WHERE public_key = ?", publicKey(types.PublicKey{1})).Error if err != nil { @@ -1310,14 +1365,14 @@ func newTestHostCheck() api.HostCheck { Prices: .7, }, Usability: api.HostUsabilityBreakdown{ - Blocked: true, - Offline: true, - LowScore: true, - RedundantIP: true, - Gouging: true, - NotAcceptingContracts: true, - NotAnnounced: true, - NotCompletingScan: true, + Blocked: false, + Offline: false, + LowScore: false, + RedundantIP: false, + Gouging: false, + NotAcceptingContracts: false, + NotAnnounced: false, + NotCompletingScan: false, }, } } From 104731b72197354f9811a85992684de571c782fa Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 25 Mar 2024 16:17:07 +0100 Subject: [PATCH 14/23] bus: fix typo --- bus/bus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bus/bus.go b/bus/bus.go index d9eba1e96..f8bd2effa 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -778,7 +778,7 @@ func (b *bus) searchHostsHandlerPOST(jc jape.Context) { } // TODO: on the next major release: - // - properly default search params (currenlty no defaults are set) + // - properly default search params (currently no defaults are set) // - properly validate and return 400 (currently validation is done in autopilot and the store) hosts, err := b.hdb.SearchHosts(jc.Request.Context(), req.AutopilotID, req.FilterMode, req.UsabilityMode, req.AddressContains, req.KeyIn, req.Offset, req.Limit) From a6b5d16eb5774e73b2ec52e6612bdfb6d766c52d Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 25 Mar 2024 19:52:18 +0100 Subject: [PATCH 15/23] autopilot: fix response types --- api/autopilot.go | 16 ---------------- autopilot/client.go | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/api/autopilot.go b/api/autopilot.go index 22598b28c..23425aacf 100644 --- a/api/autopilot.go +++ b/api/autopilot.go @@ -4,7 +4,6 @@ import ( "errors" "go.sia.tech/core/types" - "go.sia.tech/renterd/hostdb" ) const ( @@ -119,21 +118,6 @@ type ( } Recommendation *ConfigRecommendation `json:"recommendation,omitempty"` } - - // HostHandlerResponse is the response type for the /host/:hostkey endpoint. - HostHandlerResponse struct { - Host hostdb.Host `json:"host"` - Checks *HostHandlerResponseChecks `json:"checks,omitempty"` - } - - HostHandlerResponseChecks struct { - Gouging bool `json:"gouging"` - GougingBreakdown HostGougingBreakdown `json:"gougingBreakdown"` - Score float64 `json:"score"` - ScoreBreakdown HostScoreBreakdown `json:"scoreBreakdown"` - Usable bool `json:"usable"` - UnusableReasons []string `json:"unusableReasons"` - } ) func (c AutopilotConfig) Validate() error { diff --git a/autopilot/client.go b/autopilot/client.go index f657d01fc..010c1f037 100644 --- a/autopilot/client.go +++ b/autopilot/client.go @@ -40,7 +40,7 @@ func (c *Client) HostInfo(hostKey types.PublicKey) (resp api.HostResponse, err e } // HostInfo returns information about all hosts. -func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostHandlerResponse, err error) { +func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostResponse, err error) { err = c.c.POST("/hosts", api.SearchHostsRequest{ Offset: offset, Limit: limit, From 8484bb06b6185f7052f0dfef05becf7e888dcaa8 Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 25 Mar 2024 20:03:57 +0100 Subject: [PATCH 16/23] bus: fix route --- autopilot/client.go | 2 +- bus/bus.go | 4 ++-- bus/client/hosts.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/autopilot/client.go b/autopilot/client.go index 010c1f037..b1d3a4ac6 100644 --- a/autopilot/client.go +++ b/autopilot/client.go @@ -41,7 +41,7 @@ func (c *Client) HostInfo(hostKey types.PublicKey) (resp api.HostResponse, err e // HostInfo returns information about all hosts. func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostResponse, err error) { - err = c.c.POST("/hosts", api.SearchHostsRequest{ + err = c.c.POST("/hosts", api.SearchHostOptions{ Offset: offset, Limit: limit, FilterMode: filterMode, diff --git a/bus/bus.go b/bus/bus.go index f8bd2effa..eeb833156 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -253,7 +253,7 @@ func (b *bus) Handler() http.Handler { "GET /autopilot/:id": b.autopilotsHandlerGET, "PUT /autopilot/:id": b.autopilotsHandlerPUT, - "PUT /autopilot/:id/host/:hostkey/checks": b.autopilotHostChecksHandlerPUT, + "PUT /autopilot/:id/host/:hostkey/check": b.autopilotHostCheckHandlerPUT, "GET /buckets": b.bucketsHandlerGET, "POST /buckets": b.bucketsHandlerPOST, @@ -1969,7 +1969,7 @@ func (b *bus) autopilotsHandlerPUT(jc jape.Context) { jc.Check("failed to update autopilot", b.as.UpdateAutopilot(jc.Request.Context(), ap)) } -func (b *bus) autopilotHostChecksHandlerPUT(jc jape.Context) { +func (b *bus) autopilotHostCheckHandlerPUT(jc jape.Context) { var id string if jc.DecodeParam("id", &id) != nil { return diff --git a/bus/client/hosts.go b/bus/client/hosts.go index 4e1aeab30..f1f072b15 100644 --- a/bus/client/hosts.go +++ b/bus/client/hosts.go @@ -106,6 +106,6 @@ func (c *Client) UpdateHostBlocklist(ctx context.Context, add, remove []string, // UpdateHostCheck updates the host with the most recent check performed by the // autopilot with given id. func (c *Client) UpdateHostCheck(ctx context.Context, autopilotID string, hostKey types.PublicKey, hostCheck api.HostCheck) (err error) { - err = c.c.WithContext(ctx).PUT(fmt.Sprintf("/autopilot/%s/host/%s", autopilotID, hostKey), hostCheck) + err = c.c.WithContext(ctx).PUT(fmt.Sprintf("/autopilot/%s/host/%s/check", autopilotID, hostKey), hostCheck) return } From 65c330d7dd19cd7bdf07adb11314f3c9b61fd19d Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 25 Mar 2024 20:12:41 +0100 Subject: [PATCH 17/23] stores: add retry to UpdateHostCheck --- stores/hostdb.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stores/hostdb.go b/stores/hostdb.go index a121ab7d0..822e97f95 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -505,7 +505,7 @@ func (ss *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (api.Host } func (ss *SQLStore) UpdateHostCheck(ctx context.Context, autopilotID string, hk types.PublicKey, hc api.HostCheck) (err error) { - err = ss.db.Transaction(func(tx *gorm.DB) error { + err = ss.retryTransaction(ctx, (func(tx *gorm.DB) error { // fetch ap id var apID uint if err := tx. @@ -566,7 +566,7 @@ func (ss *SQLStore) UpdateHostCheck(ctx context.Context, autopilotID string, hk GougingUploadErr: hc.Gouging.UploadErr, }). Error - }) + })) return } From 69830362ecb240d0c4cf19fb77de6590bc0b986e Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 25 Mar 2024 20:19:04 +0100 Subject: [PATCH 18/23] autopilot: fix SearchHostOptions usage --- api/host.go | 19 ---------------- autopilot/autopilot.go | 50 +++++++++++++++++++++++++++++++++++------- autopilot/client.go | 2 +- 3 files changed, 43 insertions(+), 28 deletions(-) diff --git a/api/host.go b/api/host.go index 46a5597b3..c1e72aa16 100644 --- a/api/host.go +++ b/api/host.go @@ -258,22 +258,3 @@ func (ub HostUsabilityBreakdown) UnusableReasons() []string { } return reasons } - -func (h Host) ToHostResponse(autopilotID string) HostResponse { - check, ok := h.Checks[autopilotID] - if !ok { - return HostResponse{Host: h.Host} - } - - return HostResponse{ - Host: h.Host, - Checks: &HostChecks{ - Gouging: check.Gouging.Gouging(), - GougingBreakdown: check.Gouging, - Score: check.Score.Score(), - ScoreBreakdown: check.Score, - Usable: check.Usability.IsUsable(), - UnusableReasons: check.Usability.UnusableReasons(), - }, - } -} diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 97ea4e939..6f2548487 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -698,20 +698,32 @@ func (ap *Autopilot) hostHandlerGET(jc jape.Context) { return } - jc.Encode(hi.ToHostResponse(ap.id)) + check, ok := hi.Checks[ap.id] + if ok { + jc.Encode(api.HostResponse{ + Host: hi.Host, + Checks: &api.HostChecks{ + Gouging: check.Gouging.Gouging(), + GougingBreakdown: check.Gouging, + Score: check.Score.Score(), + ScoreBreakdown: check.Score, + Usable: check.Usability.IsUsable(), + UnusableReasons: check.Usability.UnusableReasons(), + }, + }) + return + } + + jc.Encode(api.HostResponse{Host: hi.Host}) } func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { - var req api.SearchHostOptions + var req api.SearchHostsRequest if jc.Decode(&req) != nil { return } else if req.AutopilotID != "" && req.AutopilotID != ap.id { jc.Error(errors.New("invalid autopilot id"), http.StatusBadRequest) return - } else { - // TODO: on next major release we should not re-use options between bus - // and autopilot API if we don't support all fields in both - req.AutopilotID = ap.id } // TODO: remove on next major release @@ -719,13 +731,35 @@ func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { return } - hosts, err := ap.bus.SearchHosts(jc.Request.Context(), req) + hosts, err := ap.bus.SearchHosts(jc.Request.Context(), api.SearchHostOptions{ + AutopilotID: ap.id, + Offset: req.Offset, + Limit: req.Limit, + FilterMode: req.FilterMode, + UsabilityMode: req.UsabilityMode, + AddressContains: req.AddressContains, + KeyIn: req.KeyIn, + }) if jc.Check("failed to get host info", err) != nil { return } resps := make([]api.HostResponse, len(hosts)) for i, host := range hosts { - resps[i] = host.ToHostResponse(ap.id) + if check, ok := host.Checks[ap.id]; ok { + resps[i] = api.HostResponse{ + Host: host.Host, + Checks: &api.HostChecks{ + Gouging: check.Gouging.Gouging(), + GougingBreakdown: check.Gouging, + Score: check.Score.Score(), + ScoreBreakdown: check.Score, + Usable: check.Usability.IsUsable(), + UnusableReasons: check.Usability.UnusableReasons(), + }, + } + } else { + resps[i] = api.HostResponse{Host: host.Host} + } } jc.Encode(resps) } diff --git a/autopilot/client.go b/autopilot/client.go index b1d3a4ac6..010c1f037 100644 --- a/autopilot/client.go +++ b/autopilot/client.go @@ -41,7 +41,7 @@ func (c *Client) HostInfo(hostKey types.PublicKey) (resp api.HostResponse, err e // HostInfo returns information about all hosts. func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostResponse, err error) { - err = c.c.POST("/hosts", api.SearchHostOptions{ + err = c.c.POST("/hosts", api.SearchHostsRequest{ Offset: offset, Limit: limit, FilterMode: filterMode, From d3c428362bd4409594d4e11c9167b43890e7a0fe Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 25 Mar 2024 20:33:39 +0100 Subject: [PATCH 19/23] hostdb: move host to api package --- api/host.go | 76 ++++++++++++++++++++++++++++--- autopilot/autopilot.go | 11 ++--- autopilot/autopilot_test.go | 47 +++++++++---------- autopilot/contractor.go | 11 ++--- autopilot/host_test.go | 14 +++--- autopilot/hostfilter.go | 2 +- autopilot/hosts_test.go | 8 ++-- autopilot/hostscore.go | 19 ++++---- autopilot/hostscore_test.go | 3 +- autopilot/scanner.go | 3 +- autopilot/scanner_test.go | 19 +++----- autopilot/workerpool.go | 3 +- bus/bus.go | 7 ++- bus/client/hosts.go | 7 ++- hostdb/hostdb.go | 71 ----------------------------- internal/test/e2e/cluster_test.go | 5 +- internal/test/e2e/pruning_test.go | 5 +- stores/hostdb.go | 58 ++++++++++++----------- stores/hostdb_test.go | 30 ++++++------ worker/client/rhp.go | 3 +- worker/host.go | 9 ++-- worker/host_test.go | 11 ++--- worker/interactions.go | 6 +-- worker/mocks_test.go | 11 ++--- worker/pricetables.go | 18 ++++---- worker/pricetables_test.go | 3 +- worker/rhpv3.go | 21 ++++----- worker/worker.go | 11 ++--- 28 files changed, 227 insertions(+), 265 deletions(-) diff --git a/api/host.go b/api/host.go index c1e72aa16..e3a414dfb 100644 --- a/api/host.go +++ b/api/host.go @@ -5,9 +5,11 @@ import ( "fmt" "net/url" "strings" + "time" + rhpv2 "go.sia.tech/core/rhp/v2" + rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" - "go.sia.tech/renterd/hostdb" ) const ( @@ -41,12 +43,12 @@ var ( type ( // HostsScanRequest is the request type for the /hosts/scans endpoint. HostsScanRequest struct { - Scans []hostdb.HostScan `json:"scans"` + Scans []HostScan `json:"scans"` } // HostsPriceTablesRequest is the request type for the /hosts/pricetables endpoint. HostsPriceTablesRequest struct { - PriceTableUpdates []hostdb.PriceTableUpdate `json:"priceTableUpdates"` + PriceTableUpdates []HostPriceTableUpdate `json:"priceTableUpdates"` } // HostsRemoveRequest is the request type for the /hosts/remove endpoint. @@ -70,7 +72,7 @@ type ( // HostResponse is the response type for the GET // /api/autopilot/host/:hostkey endpoint. HostResponse struct { - Host hostdb.Host `json:"host"` + Host Host `json:"host"` Checks *HostChecks `json:"checks,omitempty"` } @@ -146,9 +148,54 @@ func (opts HostsForScanningOptions) Apply(values url.Values) { type ( Host struct { - hostdb.Host - Blocked bool `json:"blocked"` - Checks map[string]HostCheck `json:"checks"` + KnownSince time.Time `json:"knownSince"` + LastAnnouncement time.Time `json:"lastAnnouncement"` + PublicKey types.PublicKey `json:"publicKey"` + NetAddress string `json:"netAddress"` + PriceTable HostPriceTable `json:"priceTable"` + Settings rhpv2.HostSettings `json:"settings"` + Interactions HostInteractions `json:"interactions"` + Scanned bool `json:"scanned"` + Blocked bool `json:"blocked"` + Checks map[string]HostCheck `json:"checks"` + } + + HostAddress struct { + PublicKey types.PublicKey `json:"publicKey"` + NetAddress string `json:"netAddress"` + } + + HostInteractions struct { + TotalScans uint64 `json:"totalScans"` + LastScan time.Time `json:"lastScan"` + LastScanSuccess bool `json:"lastScanSuccess"` + LostSectors uint64 `json:"lostSectors"` + SecondToLastScanSuccess bool `json:"secondToLastScanSuccess"` + Uptime time.Duration `json:"uptime"` + Downtime time.Duration `json:"downtime"` + + SuccessfulInteractions float64 `json:"successfulInteractions"` + FailedInteractions float64 `json:"failedInteractions"` + } + + HostScan struct { + HostKey types.PublicKey `json:"hostKey"` + Success bool + Timestamp time.Time + Settings rhpv2.HostSettings + PriceTable rhpv3.HostPriceTable + } + + HostPriceTable struct { + rhpv3.HostPriceTable + Expiry time.Time `json:"expiry"` + } + + HostPriceTableUpdate struct { + HostKey types.PublicKey `json:"hostKey"` + Success bool + Timestamp time.Time + PriceTable HostPriceTable } HostCheck struct { @@ -187,6 +234,21 @@ type ( } ) +// IsAnnounced returns whether the host has been announced. +func (h Host) IsAnnounced() bool { + return !h.LastAnnouncement.IsZero() +} + +// IsOnline returns whether a host is considered online. +func (h Host) IsOnline() bool { + if h.Interactions.TotalScans == 0 { + return false + } else if h.Interactions.TotalScans == 1 { + return h.Interactions.LastScanSuccess + } + return h.Interactions.LastScanSuccess || h.Interactions.SecondToLastScanSuccess +} + func (sb HostScoreBreakdown) String() string { return fmt.Sprintf("Age: %v, Col: %v, Int: %v, SR: %v, UT: %v, V: %v, Pr: %v", sb.Age, sb.Collateral, sb.Interactions, sb.StorageRemaining, sb.Uptime, sb.Version, sb.Prices) } diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 6f2548487..4e3023a2f 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -17,7 +17,6 @@ import ( "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" "go.sia.tech/renterd/build" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/object" "go.sia.tech/renterd/wallet" @@ -54,7 +53,7 @@ type Bus interface { // hostdb Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) - HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error) + HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]api.HostAddress, error) RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.Host, error) UpdateHostCheck(ctx context.Context, autopilotID string, hostKey types.PublicKey, hostCheck api.HostCheck) error @@ -701,7 +700,7 @@ func (ap *Autopilot) hostHandlerGET(jc jape.Context) { check, ok := hi.Checks[ap.id] if ok { jc.Encode(api.HostResponse{ - Host: hi.Host, + Host: hi, Checks: &api.HostChecks{ Gouging: check.Gouging.Gouging(), GougingBreakdown: check.Gouging, @@ -714,7 +713,7 @@ func (ap *Autopilot) hostHandlerGET(jc jape.Context) { return } - jc.Encode(api.HostResponse{Host: hi.Host}) + jc.Encode(api.HostResponse{Host: hi}) } func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { @@ -747,7 +746,7 @@ func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { for i, host := range hosts { if check, ok := host.Checks[ap.id]; ok { resps[i] = api.HostResponse{ - Host: host.Host, + Host: host, Checks: &api.HostChecks{ Gouging: check.Gouging.Gouging(), GougingBreakdown: check.Gouging, @@ -758,7 +757,7 @@ func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { }, } } else { - resps[i] = api.HostResponse{Host: host.Host} + resps[i] = api.HostResponse{Host: host} } } jc.Encode(resps) diff --git a/autopilot/autopilot_test.go b/autopilot/autopilot_test.go index a21b55c7b..a572c56fc 100644 --- a/autopilot/autopilot_test.go +++ b/autopilot/autopilot_test.go @@ -9,7 +9,6 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" ) func TestOptimiseGougingSetting(t *testing.T) { @@ -18,32 +17,30 @@ func TestOptimiseGougingSetting(t *testing.T) { for i := 0; i < 10; i++ { hosts = append(hosts, api.Host{ - Host: hostdb.Host{ - KnownSince: time.Unix(0, 0), - PriceTable: hostdb.HostPriceTable{ - HostPriceTable: rhpv3.HostPriceTable{ - CollateralCost: types.Siacoins(1), - MaxCollateral: types.Siacoins(1000), - }, + KnownSince: time.Unix(0, 0), + PriceTable: api.HostPriceTable{ + HostPriceTable: rhpv3.HostPriceTable{ + CollateralCost: types.Siacoins(1), + MaxCollateral: types.Siacoins(1000), }, - Settings: rhpv2.HostSettings{ - AcceptingContracts: true, - Collateral: types.Siacoins(1), - MaxCollateral: types.Siacoins(1000), - Version: "1.6.0", - }, - Interactions: hostdb.Interactions{ - Uptime: time.Hour * 1000, - LastScan: time.Now(), - LastScanSuccess: true, - SecondToLastScanSuccess: true, - TotalScans: 100, - }, - LastAnnouncement: time.Unix(0, 0), - Scanned: true, }, - Blocked: false, - Checks: nil, + Settings: rhpv2.HostSettings{ + AcceptingContracts: true, + Collateral: types.Siacoins(1), + MaxCollateral: types.Siacoins(1000), + Version: "1.6.0", + }, + Interactions: api.HostInteractions{ + Uptime: time.Hour * 1000, + LastScan: time.Now(), + LastScanSuccess: true, + SecondToLastScanSuccess: true, + TotalScans: 100, + }, + LastAnnouncement: time.Unix(0, 0), + Scanned: true, + Blocked: false, + Checks: nil, }) } diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 0e85e4302..e72b0078e 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -15,7 +15,6 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/wallet" "go.sia.tech/renterd/worker" @@ -103,7 +102,7 @@ type ( } scoredHost struct { - host hostdb.Host + host api.Host score float64 } @@ -761,7 +760,7 @@ func (c *contractor) runContractChecks(ctx context.Context, contracts []api.Cont } // decide whether the contract is still good - ci := contractInfo{contract: contract, priceTable: host.Host.PriceTable.HostPriceTable, settings: host.Host.Settings} + ci := contractInfo{contract: contract, priceTable: host.PriceTable.HostPriceTable, settings: host.Settings} usable, recoverable, refresh, renew, reasons := c.isUsableContract(state.cfg, state, ci, bh, ipFilter) ci.usable = usable ci.recoverable = recoverable @@ -1323,7 +1322,7 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []api.Host, usedH h.PriceTable.HostBlockHeight = cs.BlockHeight hc := checkHost(state.cfg, state.rs, gc, h, minScore, storedData[h.PublicKey]) if hc.Usability.IsUsable() { - candidates = append(candidates, scoredHost{h.Host, hc.Score.Score()}) + candidates = append(candidates, scoredHost{h, hc.Score.Score()}) continue } @@ -1505,7 +1504,7 @@ func (c *contractor) refreshContract(ctx context.Context, w Worker, ci contractI return refreshedContract, true, nil } -func (c *contractor) formContract(ctx context.Context, w Worker, host hostdb.Host, minInitialContractFunds, maxInitialContractFunds types.Currency, budget *types.Currency) (cm api.ContractMetadata, proceed bool, err error) { +func (c *contractor) formContract(ctx context.Context, w Worker, host api.Host, minInitialContractFunds, maxInitialContractFunds types.Currency, budget *types.Currency) (cm api.ContractMetadata, proceed bool, err error) { // convenience variables state := c.ap.State() hk := host.PublicKey @@ -1632,7 +1631,7 @@ func initialContractFundingMinMax(cfg api.AutopilotConfig) (min types.Currency, return } -func refreshPriceTable(ctx context.Context, w Worker, host *hostdb.Host) error { +func refreshPriceTable(ctx context.Context, w Worker, host *api.Host) error { // return early if the host's pricetable is not expired yet if time.Now().Before(host.PriceTable.Expiry) { return nil diff --git a/autopilot/host_test.go b/autopilot/host_test.go index fa1a0ab44..965b2b05a 100644 --- a/autopilot/host_test.go +++ b/autopilot/host_test.go @@ -8,7 +8,7 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" - "go.sia.tech/renterd/hostdb" + "go.sia.tech/renterd/api" "lukechampine.com/frand" ) @@ -40,20 +40,20 @@ func TestHost(t *testing.T) { } } -func newTestHosts(n int) []hostdb.Host { - hosts := make([]hostdb.Host, n) +func newTestHosts(n int) []api.Host { + hosts := make([]api.Host, n) for i := 0; i < n; i++ { hosts[i] = newTestHost(randomHostKey(), newTestHostPriceTable(), newTestHostSettings()) } return hosts } -func newTestHost(hk types.PublicKey, pt rhpv3.HostPriceTable, settings rhpv2.HostSettings) hostdb.Host { - return hostdb.Host{ +func newTestHost(hk types.PublicKey, pt rhpv3.HostPriceTable, settings rhpv2.HostSettings) api.Host { + return api.Host{ NetAddress: randomIP().String(), KnownSince: time.Now(), LastAnnouncement: time.Now(), - Interactions: hostdb.Interactions{ + Interactions: api.HostInteractions{ TotalScans: 2, LastScan: time.Now().Add(-time.Minute), LastScanSuccess: true, @@ -65,7 +65,7 @@ func newTestHost(hk types.PublicKey, pt rhpv3.HostPriceTable, settings rhpv2.Hos FailedInteractions: 0, }, PublicKey: hk, - PriceTable: hostdb.HostPriceTable{HostPriceTable: pt, Expiry: time.Now().Add(time.Minute)}, + PriceTable: api.HostPriceTable{HostPriceTable: pt, Expiry: time.Now().Add(time.Minute)}, Settings: settings, Scanned: true, } diff --git a/autopilot/hostfilter.go b/autopilot/hostfilter.go index 9a6e4afbf..61edbbe69 100644 --- a/autopilot/hostfilter.go +++ b/autopilot/hostfilter.go @@ -140,7 +140,7 @@ func checkHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.Gou // not gouging, this because the core package does not have overflow // checks in its cost calculations needed to calculate the period // cost - sb = hostScore(cfg, h.Host, storedData, rs.Redundancy()) + sb = hostScore(cfg, h, storedData, rs.Redundancy()) if sb.Score() < minScore { ub.LowScore = true } diff --git a/autopilot/hosts_test.go b/autopilot/hosts_test.go index 332bf1ea3..6644a1cd2 100644 --- a/autopilot/hosts_test.go +++ b/autopilot/hosts_test.go @@ -5,7 +5,7 @@ import ( "testing" "go.sia.tech/core/types" - "go.sia.tech/renterd/hostdb" + "go.sia.tech/renterd/api" "lukechampine.com/frand" ) @@ -18,7 +18,7 @@ func TestScoredHostsRandSelectByScore(t *testing.T) { var hosts scoredHosts for hk, score := range hostToScores { - hosts = append(hosts, scoredHost{score: score, host: hostdb.Host{PublicKey: hk}}) + hosts = append(hosts, scoredHost{score: score, host: api.Host{PublicKey: hk}}) } for i := 0; i < 1000; i++ { @@ -55,8 +55,8 @@ func TestScoredHostsRandSelectByScore(t *testing.T) { // assert select is random on equal inputs counts := make([]int, 2) hosts = scoredHosts{ - {score: .1, host: hostdb.Host{PublicKey: types.PublicKey{1}}}, - {score: .1, host: hostdb.Host{PublicKey: types.PublicKey{2}}}, + {score: .1, host: api.Host{PublicKey: types.PublicKey{1}}}, + {score: .1, host: api.Host{PublicKey: types.PublicKey{2}}}, } for i := 0; i < 100; i++ { if hosts.randSelectByScore(1)[0].host.PublicKey == (types.PublicKey{1}) { diff --git a/autopilot/hostscore.go b/autopilot/hostscore.go index 3c26dce42..fc98499f1 100644 --- a/autopilot/hostscore.go +++ b/autopilot/hostscore.go @@ -9,13 +9,12 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/siad/build" ) const smallestValidScore = math.SmallestNonzeroFloat64 -func hostScore(cfg api.AutopilotConfig, h hostdb.Host, storedData uint64, expectedRedundancy float64) api.HostScoreBreakdown { +func hostScore(cfg api.AutopilotConfig, h api.Host, storedData uint64, expectedRedundancy float64) api.HostScoreBreakdown { // idealDataPerHost is the amount of data that we would have to put on each // host assuming that our storage requirements were spread evenly across // every single host. @@ -92,7 +91,7 @@ func storageRemainingScore(h rhpv2.HostSettings, storedData uint64, allocationPe return math.Pow(storageRatio, 2.0) } -func ageScore(h hostdb.Host) float64 { +func ageScore(h api.Host) float64 { // sanity check if h.KnownSince.IsZero() { return 0 @@ -179,14 +178,14 @@ func collateralScore(cfg api.AutopilotConfig, pt rhpv3.HostPriceTable, allocatio } } -func interactionScore(h hostdb.Host) float64 { +func interactionScore(h api.Host) float64 { success, fail := 30.0, 1.0 success += h.Interactions.SuccessfulInteractions fail += h.Interactions.FailedInteractions return math.Pow(success/(success+fail), 10) } -func uptimeScore(h hostdb.Host) float64 { +func uptimeScore(h api.Host) float64 { secondToLastScanSuccess := h.Interactions.SecondToLastScanSuccess lastScanSuccess := h.Interactions.LastScanSuccess uptime := h.Interactions.Uptime @@ -258,7 +257,7 @@ func versionScore(settings rhpv2.HostSettings) float64 { // contractPriceForScore returns the contract price of the host used for // scoring. Since we don't know whether rhpv2 or rhpv3 are used, we return the // bigger one for a pesimistic score. -func contractPriceForScore(h hostdb.Host) types.Currency { +func contractPriceForScore(h api.Host) types.Currency { cp := h.Settings.ContractPrice if cp.Cmp(h.PriceTable.ContractPrice) > 0 { cp = h.PriceTable.ContractPrice @@ -285,26 +284,26 @@ func sectorUploadCost(pt rhpv3.HostPriceTable, duration uint64) types.Currency { return uploadSectorCostRHPv3 } -func uploadCostForScore(cfg api.AutopilotConfig, h hostdb.Host, bytes uint64) types.Currency { +func uploadCostForScore(cfg api.AutopilotConfig, h api.Host, bytes uint64) types.Currency { uploadSectorCostRHPv3 := sectorUploadCost(h.PriceTable.HostPriceTable, cfg.Contracts.Period) numSectors := bytesToSectors(bytes) return uploadSectorCostRHPv3.Mul64(numSectors) } -func downloadCostForScore(h hostdb.Host, bytes uint64) types.Currency { +func downloadCostForScore(h api.Host, bytes uint64) types.Currency { rsc := h.PriceTable.BaseCost().Add(h.PriceTable.ReadSectorCost(rhpv2.SectorSize)) downloadSectorCostRHPv3, _ := rsc.Total() numSectors := bytesToSectors(bytes) return downloadSectorCostRHPv3.Mul64(numSectors) } -func storageCostForScore(cfg api.AutopilotConfig, h hostdb.Host, bytes uint64) types.Currency { +func storageCostForScore(cfg api.AutopilotConfig, h api.Host, bytes uint64) types.Currency { storeSectorCostRHPv3 := sectorStorageCost(h.PriceTable.HostPriceTable, cfg.Contracts.Period) numSectors := bytesToSectors(bytes) return storeSectorCostRHPv3.Mul64(numSectors) } -func hostPeriodCostForScore(h hostdb.Host, cfg api.AutopilotConfig, expectedRedundancy float64) types.Currency { +func hostPeriodCostForScore(h api.Host, cfg api.AutopilotConfig, expectedRedundancy float64) types.Currency { // compute how much data we upload, download and store. uploadPerHost := uint64(float64(cfg.Contracts.Upload) * expectedRedundancy / float64(cfg.Contracts.Amount)) downloadPerHost := uint64(float64(cfg.Contracts.Download) * expectedRedundancy / float64(cfg.Contracts.Amount)) diff --git a/autopilot/hostscore_test.go b/autopilot/hostscore_test.go index e48417235..bc5f61b8a 100644 --- a/autopilot/hostscore_test.go +++ b/autopilot/hostscore_test.go @@ -9,7 +9,6 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" ) var cfg = api.AutopilotConfig{ @@ -34,7 +33,7 @@ var cfg = api.AutopilotConfig{ func TestHostScore(t *testing.T) { day := 24 * time.Hour - newHost := func(s rhpv2.HostSettings) hostdb.Host { + newHost := func(s rhpv2.HostSettings) api.Host { return newTestHost(randomHostKey(), newTestHostPriceTable(), s) } h1 := newHost(newTestHostSettings()) diff --git a/autopilot/scanner.go b/autopilot/scanner.go index d733c8d0c..f466a842d 100644 --- a/autopilot/scanner.go +++ b/autopilot/scanner.go @@ -11,7 +11,6 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/internal/utils" "go.uber.org/zap" ) @@ -32,7 +31,7 @@ type ( // scanner tests with every interface change bus interface { SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.Host, error) - HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error) + HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]api.HostAddress, error) RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) } diff --git a/autopilot/scanner_test.go b/autopilot/scanner_test.go index 860a855fe..512cbd517 100644 --- a/autopilot/scanner_test.go +++ b/autopilot/scanner_test.go @@ -9,13 +9,12 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) type mockBus struct { - hosts []hostdb.Host + hosts []api.Host reqs []string } @@ -32,14 +31,10 @@ func (b *mockBus) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ( end = len(b.hosts) } - hosts := make([]api.Host, len(b.hosts[start:end])) - for i, h := range b.hosts[start:end] { - hosts[i] = api.Host{Host: h} - } - return hosts, nil + return b.hosts[start:end], nil } -func (b *mockBus) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error) { +func (b *mockBus) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]api.HostAddress, error) { hosts, err := b.SearchHosts(ctx, api.SearchHostOptions{ Offset: opts.Offset, Limit: opts.Limit, @@ -47,9 +42,9 @@ func (b *mockBus) HostsForScanning(ctx context.Context, opts api.HostsForScannin if err != nil { return nil, err } - var hostAddresses []hostdb.HostAddress + var hostAddresses []api.HostAddress for _, h := range hosts { - hostAddresses = append(hostAddresses, hostdb.HostAddress{ + hostAddresses = append(hostAddresses, api.HostAddress{ NetAddress: h.NetAddress, PublicKey: h.PublicKey, }) @@ -80,8 +75,8 @@ func (w *mockWorker) RHPScan(ctx context.Context, hostKey types.PublicKey, hostI return api.RHPScanResponse{}, nil } -func (w *mockWorker) RHPPriceTable(ctx context.Context, hostKey types.PublicKey, siamuxAddr string) (hostdb.HostPriceTable, error) { - return hostdb.HostPriceTable{}, nil +func (w *mockWorker) RHPPriceTable(ctx context.Context, hostKey types.PublicKey, siamuxAddr string) (api.HostPriceTable, error) { + return api.HostPriceTable{}, nil } func TestScanner(t *testing.T) { diff --git a/autopilot/workerpool.go b/autopilot/workerpool.go index d8c821354..16a6b4c99 100644 --- a/autopilot/workerpool.go +++ b/autopilot/workerpool.go @@ -9,7 +9,6 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/object" "lukechampine.com/frand" ) @@ -23,7 +22,7 @@ type Worker interface { RHPBroadcast(ctx context.Context, fcid types.FileContractID) (err error) RHPForm(ctx context.Context, endHeight uint64, hk types.PublicKey, hostIP string, renterAddress types.Address, renterFunds types.Currency, hostCollateral types.Currency) (rhpv2.ContractRevision, []types.Transaction, error) RHPFund(ctx context.Context, contractID types.FileContractID, hostKey types.PublicKey, hostIP, siamuxAddr string, balance types.Currency) (err error) - RHPPriceTable(ctx context.Context, hostKey types.PublicKey, siamuxAddr string, timeout time.Duration) (hostdb.HostPriceTable, error) + RHPPriceTable(ctx context.Context, hostKey types.PublicKey, siamuxAddr string, timeout time.Duration) (api.HostPriceTable, error) RHPPruneContract(ctx context.Context, fcid types.FileContractID, timeout time.Duration) (pruned, remaining uint64, err error) RHPRenew(ctx context.Context, fcid types.FileContractID, endHeight uint64, hk types.PublicKey, hostIP string, hostAddress, renterAddress types.Address, renterFunds, minNewCollateral types.Currency, expectedStorage, windowSize uint64) (api.RHPRenewResponse, error) RHPScan(ctx context.Context, hostKey types.PublicKey, hostIP string, timeout time.Duration) (api.RHPScanResponse, error) diff --git a/bus/bus.go b/bus/bus.go index eeb833156..24f319de6 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -23,7 +23,6 @@ import ( "go.sia.tech/renterd/api" "go.sia.tech/renterd/build" "go.sia.tech/renterd/bus/client" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/object" "go.sia.tech/renterd/wallet" "go.sia.tech/renterd/webhooks" @@ -94,9 +93,9 @@ type ( Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) HostAllowlist(ctx context.Context) ([]types.PublicKey, error) HostBlocklist(ctx context.Context) ([]string, error) - HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]hostdb.HostAddress, error) - RecordHostScans(ctx context.Context, scans []hostdb.HostScan) error - RecordPriceTables(ctx context.Context, priceTableUpdate []hostdb.PriceTableUpdate) error + HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]api.HostAddress, error) + RecordHostScans(ctx context.Context, scans []api.HostScan) error + RecordPriceTables(ctx context.Context, priceTableUpdate []api.HostPriceTableUpdate) error RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) ResetLostSectors(ctx context.Context, hk types.PublicKey) error SearchHosts(ctx context.Context, autopilotID, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.Host, error) diff --git a/bus/client/hosts.go b/bus/client/hosts.go index f1f072b15..709cb899c 100644 --- a/bus/client/hosts.go +++ b/bus/client/hosts.go @@ -8,7 +8,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" ) // Host returns information about a particular host known to the server. @@ -39,7 +38,7 @@ func (c *Client) Hosts(ctx context.Context, opts api.GetHostsOptions) (hosts []a // HostsForScanning returns 'limit' host addresses at given 'offset' which // haven't been scanned after lastScan. -func (c *Client) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) (hosts []hostdb.HostAddress, err error) { +func (c *Client) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) (hosts []api.HostAddress, err error) { values := url.Values{} opts.Apply(values) err = c.c.WithContext(ctx).GET("/hosts/scanning?"+values.Encode(), &hosts) @@ -47,7 +46,7 @@ func (c *Client) HostsForScanning(ctx context.Context, opts api.HostsForScanning } // RecordHostInteraction records an interaction for the supplied host. -func (c *Client) RecordHostScans(ctx context.Context, scans []hostdb.HostScan) (err error) { +func (c *Client) RecordHostScans(ctx context.Context, scans []api.HostScan) (err error) { err = c.c.WithContext(ctx).POST("/hosts/scans", api.HostsScanRequest{ Scans: scans, }, nil) @@ -55,7 +54,7 @@ func (c *Client) RecordHostScans(ctx context.Context, scans []hostdb.HostScan) ( } // RecordHostInteraction records an interaction for the supplied host. -func (c *Client) RecordPriceTables(ctx context.Context, priceTableUpdates []hostdb.PriceTableUpdate) (err error) { +func (c *Client) RecordPriceTables(ctx context.Context, priceTableUpdates []api.HostPriceTableUpdate) (err error) { err = c.c.WithContext(ctx).POST("/hosts/pricetables", api.HostsPriceTablesRequest{ PriceTableUpdates: priceTableUpdates, }, nil) diff --git a/hostdb/hostdb.go b/hostdb/hostdb.go index 1f4c341de..1a957e327 100644 --- a/hostdb/hostdb.go +++ b/hostdb/hostdb.go @@ -4,8 +4,6 @@ import ( "time" "gitlab.com/NebulousLabs/encoding" - rhpv2 "go.sia.tech/core/rhp/v2" - rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/siad/crypto" "go.sia.tech/siad/modules" @@ -59,72 +57,3 @@ func ForEachAnnouncement(b types.Block, height uint64, fn func(types.PublicKey, } } } - -// Interactions contains metadata about a host's interactions. -type Interactions struct { - TotalScans uint64 `json:"totalScans"` - LastScan time.Time `json:"lastScan"` - LastScanSuccess bool `json:"lastScanSuccess"` - LostSectors uint64 `json:"lostSectors"` - SecondToLastScanSuccess bool `json:"secondToLastScanSuccess"` - Uptime time.Duration `json:"uptime"` - Downtime time.Duration `json:"downtime"` - - SuccessfulInteractions float64 `json:"successfulInteractions"` - FailedInteractions float64 `json:"failedInteractions"` -} - -type HostScan struct { - HostKey types.PublicKey `json:"hostKey"` - Success bool - Timestamp time.Time - Settings rhpv2.HostSettings - PriceTable rhpv3.HostPriceTable -} - -type PriceTableUpdate struct { - HostKey types.PublicKey `json:"hostKey"` - Success bool - Timestamp time.Time - PriceTable HostPriceTable -} - -// HostAddress contains the address of a specific host identified by a public -// key. -type HostAddress struct { - PublicKey types.PublicKey `json:"publicKey"` - NetAddress string `json:"netAddress"` -} - -// A Host pairs a host's public key with a set of interactions. -type Host struct { - KnownSince time.Time `json:"knownSince"` - LastAnnouncement time.Time `json:"lastAnnouncement"` - PublicKey types.PublicKey `json:"publicKey"` - NetAddress string `json:"netAddress"` - PriceTable HostPriceTable `json:"priceTable"` - Settings rhpv2.HostSettings `json:"settings"` - Interactions Interactions `json:"interactions"` - Scanned bool `json:"scanned"` -} - -// A HostPriceTable extends the host price table with its expiry. -type HostPriceTable struct { - rhpv3.HostPriceTable - Expiry time.Time `json:"expiry"` -} - -// IsAnnounced returns whether the host has been announced. -func (h Host) IsAnnounced() bool { - return !h.LastAnnouncement.IsZero() -} - -// IsOnline returns whether a host is considered online. -func (h Host) IsOnline() bool { - if h.Interactions.TotalScans == 0 { - return false - } else if h.Interactions.TotalScans == 1 { - return h.Interactions.LastScanSuccess - } - return h.Interactions.LastScanSuccess || h.Interactions.SecondToLastScanSuccess -} diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 77898d4cf..76e6dd814 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -23,7 +23,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/internal/test" "go.sia.tech/renterd/object" "go.sia.tech/renterd/wallet" @@ -166,7 +165,7 @@ func TestNewTestCluster(t *testing.T) { if len(hi.Checks.UnusableReasons) != 0 { t.Fatal("usable hosts don't have any reasons set") } - if reflect.DeepEqual(hi.Host, hostdb.Host{}) { + if reflect.DeepEqual(hi.Host, api.Host{}) { t.Fatal("host wasn't set") } } @@ -188,7 +187,7 @@ func TestNewTestCluster(t *testing.T) { if len(hi.Checks.UnusableReasons) != 0 { t.Fatal("usable hosts don't have any reasons set") } - if reflect.DeepEqual(hi.Host, hostdb.Host{}) { + if reflect.DeepEqual(hi.Host, api.Host{}) { t.Fatal("host wasn't set") } allHosts[hi.Host.PublicKey] = struct{}{} diff --git a/internal/test/e2e/pruning_test.go b/internal/test/e2e/pruning_test.go index de948c970..7c1a856f1 100644 --- a/internal/test/e2e/pruning_test.go +++ b/internal/test/e2e/pruning_test.go @@ -11,7 +11,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/internal/test" ) @@ -32,10 +31,10 @@ func TestHostPruning(t *testing.T) { now := time.Now() recordFailedInteractions := func(n int, hk types.PublicKey) { t.Helper() - his := make([]hostdb.HostScan, n) + his := make([]api.HostScan, n) for i := 0; i < n; i++ { now = now.Add(time.Hour).Add(time.Minute) // 1m leeway - his[i] = hostdb.HostScan{ + his[i] = api.HostScan{ HostKey: hk, Timestamp: now, Success: false, diff --git a/stores/hostdb.go b/stores/hostdb.go index 822e97f95..38dde848c 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -41,7 +41,7 @@ var ( ) type ( - // dbHost defines a hostdb.Interaction as persisted in the DB. Deleting a + // dbHost defines a api.Interaction as persisted in the DB. Deleting a // host from the db will cascade the deletion and also delete the // corresponding announcements and interactions with that host. // @@ -322,31 +322,29 @@ func (h dbHost) convert(blocked bool) api.Host { checks[check.DBAutopilot.Identifier] = check.convert() } return api.Host{ - Host: hostdb.Host{ - KnownSince: h.CreatedAt, - LastAnnouncement: h.LastAnnouncement, - NetAddress: h.NetAddress, - Interactions: hostdb.Interactions{ - TotalScans: h.TotalScans, - LastScan: lastScan, - LastScanSuccess: h.LastScanSuccess, - SecondToLastScanSuccess: h.SecondToLastScanSuccess, - Uptime: h.Uptime, - Downtime: h.Downtime, - SuccessfulInteractions: h.SuccessfulInteractions, - FailedInteractions: h.FailedInteractions, - LostSectors: h.LostSectors, - }, - PriceTable: hostdb.HostPriceTable{ - HostPriceTable: h.PriceTable.convert(), - Expiry: h.PriceTableExpiry.Time, - }, - PublicKey: types.PublicKey(h.PublicKey), - Scanned: h.Scanned, - Settings: h.Settings.convert(), + KnownSince: h.CreatedAt, + LastAnnouncement: h.LastAnnouncement, + NetAddress: h.NetAddress, + Interactions: api.HostInteractions{ + TotalScans: h.TotalScans, + LastScan: lastScan, + LastScanSuccess: h.LastScanSuccess, + SecondToLastScanSuccess: h.SecondToLastScanSuccess, + Uptime: h.Uptime, + Downtime: h.Downtime, + SuccessfulInteractions: h.SuccessfulInteractions, + FailedInteractions: h.FailedInteractions, + LostSectors: h.LostSectors, }, - Blocked: blocked, - Checks: checks, + PriceTable: api.HostPriceTable{ + HostPriceTable: h.PriceTable.convert(), + Expiry: h.PriceTableExpiry.Time, + }, + PublicKey: types.PublicKey(h.PublicKey), + Scanned: h.Scanned, + Settings: h.Settings.convert(), + Blocked: blocked, + Checks: checks, } } @@ -571,7 +569,7 @@ func (ss *SQLStore) UpdateHostCheck(ctx context.Context, autopilotID string, hk } // HostsForScanning returns the address of hosts for scanning. -func (ss *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]hostdb.HostAddress, error) { +func (ss *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]api.HostAddress, error) { if offset < 0 { return nil, ErrNegativeOffset } @@ -580,7 +578,7 @@ func (ss *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time, PublicKey publicKey `gorm:"unique;index;NOT NULL"` NetAddress string } - var hostAddresses []hostdb.HostAddress + var hostAddresses []api.HostAddress err := ss.db. WithContext(ctx). @@ -591,7 +589,7 @@ func (ss *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time, Order("last_scan ASC"). FindInBatches(&hosts, hostRetrievalBatchSize, func(tx *gorm.DB, batch int) error { for _, h := range hosts { - hostAddresses = append(hostAddresses, hostdb.HostAddress{ + hostAddresses = append(hostAddresses, api.HostAddress{ PublicKey: types.PublicKey(h.PublicKey), NetAddress: h.NetAddress, }) @@ -844,7 +842,7 @@ func (ss *SQLStore) HostBlocklist(ctx context.Context) (blocklist []string, err return } -func (ss *SQLStore) RecordHostScans(ctx context.Context, scans []hostdb.HostScan) error { +func (ss *SQLStore) RecordHostScans(ctx context.Context, scans []api.HostScan) error { if len(scans) == 0 { return nil // nothing to do } @@ -965,7 +963,7 @@ func (ss *SQLStore) RecordHostScans(ctx context.Context, scans []hostdb.HostScan }) } -func (ss *SQLStore) RecordPriceTables(ctx context.Context, priceTableUpdate []hostdb.PriceTableUpdate) error { +func (ss *SQLStore) RecordPriceTables(ctx context.Context, priceTableUpdate []api.HostPriceTableUpdate) error { if len(priceTableUpdate) == 0 { return nil // nothing to do } diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index 196170b01..e810af541 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -148,7 +148,7 @@ func TestSQLHostDB(t *testing.T) { } func (s *SQLStore) addTestScan(hk types.PublicKey, t time.Time, err error, settings rhpv2.HostSettings) error { - return s.RecordHostScans(context.Background(), []hostdb.HostScan{ + return s.RecordHostScans(context.Background(), []api.HostScan{ { HostKey: hk, Settings: settings, @@ -308,15 +308,15 @@ func TestSearchHosts(t *testing.T) { t.Fatal(err) } else if len(his) != 2 { t.Fatal("unexpected") - } else if his[0].Host.PublicKey != (types.PublicKey{2}) || his[1].Host.PublicKey != (types.PublicKey{3}) { - t.Fatal("unexpected", his[0].Host.PublicKey, his[1].Host.PublicKey) + } else if his[0].PublicKey != (types.PublicKey{2}) || his[1].PublicKey != (types.PublicKey{3}) { + t.Fatal("unexpected", his[0].PublicKey, his[1].PublicKey) } his, err = ss.SearchHosts(context.Background(), "", api.HostFilterModeBlocked, api.UsabilityFilterModeAll, "", nil, 0, -1) if err != nil { t.Fatal(err) } else if len(his) != 1 { t.Fatal("unexpected") - } else if his[0].Host.PublicKey != (types.PublicKey{1}) { + } else if his[0].PublicKey != (types.PublicKey{1}) { t.Fatal("unexpected", his) } err = ss.UpdateHostBlocklistEntries(context.Background(), nil, nil, true) @@ -480,7 +480,7 @@ func TestRecordScan(t *testing.T) { if err != nil { t.Fatal(err) } - if host.Interactions != (hostdb.Interactions{}) { + if host.Interactions != (api.HostInteractions{}) { t.Fatal("mismatch") } if host.Settings != (rhpv2.HostSettings{}) { @@ -499,7 +499,7 @@ func TestRecordScan(t *testing.T) { // Record a scan. firstScanTime := time.Now().UTC() settings := rhpv2.HostSettings{NetAddress: "host.com"} - if err := ss.RecordHostScans(ctx, []hostdb.HostScan{newTestScan(hk, firstScanTime, settings, true)}); err != nil { + if err := ss.RecordHostScans(ctx, []api.HostScan{newTestScan(hk, firstScanTime, settings, true)}); err != nil { t.Fatal(err) } host, err = ss.Host(ctx, hk) @@ -514,7 +514,7 @@ func TestRecordScan(t *testing.T) { t.Fatal("wrong time") } host.Interactions.LastScan = time.Time{} - if expected := (hostdb.Interactions{ + if expected := (api.HostInteractions{ TotalScans: 1, LastScan: time.Time{}, LastScanSuccess: true, @@ -532,7 +532,7 @@ func TestRecordScan(t *testing.T) { // Record another scan 1 hour after the previous one. secondScanTime := firstScanTime.Add(time.Hour) - if err := ss.RecordHostScans(ctx, []hostdb.HostScan{newTestScan(hk, secondScanTime, settings, true)}); err != nil { + if err := ss.RecordHostScans(ctx, []api.HostScan{newTestScan(hk, secondScanTime, settings, true)}); err != nil { t.Fatal(err) } host, err = ss.Host(ctx, hk) @@ -544,7 +544,7 @@ func TestRecordScan(t *testing.T) { } host.Interactions.LastScan = time.Time{} uptime += secondScanTime.Sub(firstScanTime) - if host.Interactions != (hostdb.Interactions{ + if host.Interactions != (api.HostInteractions{ TotalScans: 2, LastScan: time.Time{}, LastScanSuccess: true, @@ -559,7 +559,7 @@ func TestRecordScan(t *testing.T) { // Record another scan 2 hours after the second one. This time it fails. thirdScanTime := secondScanTime.Add(2 * time.Hour) - if err := ss.RecordHostScans(ctx, []hostdb.HostScan{newTestScan(hk, thirdScanTime, settings, false)}); err != nil { + if err := ss.RecordHostScans(ctx, []api.HostScan{newTestScan(hk, thirdScanTime, settings, false)}); err != nil { t.Fatal(err) } host, err = ss.Host(ctx, hk) @@ -571,7 +571,7 @@ func TestRecordScan(t *testing.T) { } host.Interactions.LastScan = time.Time{} downtime += thirdScanTime.Sub(secondScanTime) - if host.Interactions != (hostdb.Interactions{ + if host.Interactions != (api.HostInteractions{ TotalScans: 3, LastScan: time.Time{}, LastScanSuccess: false, @@ -621,7 +621,7 @@ func TestRemoveHosts(t *testing.T) { hi2 := newTestScan(hk, t2, rhpv2.HostSettings{NetAddress: "host.com"}, false) // record interactions - if err := ss.RecordHostScans(context.Background(), []hostdb.HostScan{hi1, hi2}); err != nil { + if err := ss.RecordHostScans(context.Background(), []api.HostScan{hi1, hi2}); err != nil { t.Fatal(err) } @@ -649,7 +649,7 @@ func TestRemoveHosts(t *testing.T) { // record interactions t3 := now.Add(-time.Minute * 60) // 1 hour ago (60min downtime) hi3 := newTestScan(hk, t3, rhpv2.HostSettings{NetAddress: "host.com"}, false) - if err := ss.RecordHostScans(context.Background(), []hostdb.HostScan{hi3}); err != nil { + if err := ss.RecordHostScans(context.Background(), []api.HostScan{hi3}); err != nil { t.Fatal(err) } @@ -1303,8 +1303,8 @@ func hostByPubKey(tx *gorm.DB, hostKey types.PublicKey) (dbHost, error) { } // newTestScan returns a host interaction with given parameters. -func newTestScan(hk types.PublicKey, scanTime time.Time, settings rhpv2.HostSettings, success bool) hostdb.HostScan { - return hostdb.HostScan{ +func newTestScan(hk types.PublicKey, scanTime time.Time, settings rhpv2.HostSettings, success bool) api.HostScan { + return api.HostScan{ HostKey: hk, Success: success, Timestamp: scanTime, diff --git a/worker/client/rhp.go b/worker/client/rhp.go index c71ddd1dc..ec7e10b3d 100644 --- a/worker/client/rhp.go +++ b/worker/client/rhp.go @@ -8,7 +8,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" rhpv2 "go.sia.tech/core/rhp/v2" ) @@ -53,7 +52,7 @@ func (c *Client) RHPFund(ctx context.Context, contractID types.FileContractID, h } // RHPPriceTable fetches a price table for a host. -func (c *Client) RHPPriceTable(ctx context.Context, hostKey types.PublicKey, siamuxAddr string, timeout time.Duration) (pt hostdb.HostPriceTable, err error) { +func (c *Client) RHPPriceTable(ctx context.Context, hostKey types.PublicKey, siamuxAddr string, timeout time.Duration) (pt api.HostPriceTable, err error) { req := api.RHPPriceTableRequest{ HostKey: hostKey, SiamuxAddr: siamuxAddr, diff --git a/worker/host.go b/worker/host.go index cd29572cc..4f4e97496 100644 --- a/worker/host.go +++ b/worker/host.go @@ -12,7 +12,6 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.uber.org/zap" ) @@ -23,7 +22,7 @@ type ( DownloadSector(ctx context.Context, w io.Writer, root types.Hash256, offset, length uint32, overpay bool) error UploadSector(ctx context.Context, sectorRoot types.Hash256, sector *[rhpv2.SectorSize]byte, rev types.FileContractRevision) error - FetchPriceTable(ctx context.Context, rev *types.FileContractRevision) (hpt hostdb.HostPriceTable, err error) + FetchPriceTable(ctx context.Context, rev *types.FileContractRevision) (hpt api.HostPriceTable, err error) FetchRevision(ctx context.Context, fetchTimeout time.Duration) (types.FileContractRevision, error) FundAccount(ctx context.Context, balance types.Currency, rev *types.FileContractRevision) error @@ -187,12 +186,12 @@ func (h *host) RenewContract(ctx context.Context, rrr api.RHPRenewRequest) (_ rh return rev, txnSet, contractPrice, renewErr } -func (h *host) FetchPriceTable(ctx context.Context, rev *types.FileContractRevision) (hpt hostdb.HostPriceTable, err error) { +func (h *host) FetchPriceTable(ctx context.Context, rev *types.FileContractRevision) (hpt api.HostPriceTable, err error) { // fetchPT is a helper function that performs the RPC given a payment function - fetchPT := func(paymentFn PriceTablePaymentFunc) (hpt hostdb.HostPriceTable, err error) { + fetchPT := func(paymentFn PriceTablePaymentFunc) (hpt api.HostPriceTable, err error) { err = h.transportPool.withTransportV3(ctx, h.hk, h.siamuxAddr, func(ctx context.Context, t *transportV3) (err error) { hpt, err = RPCPriceTable(ctx, t, paymentFn) - h.bus.RecordPriceTables(ctx, []hostdb.PriceTableUpdate{ + h.bus.RecordPriceTables(ctx, []api.HostPriceTableUpdate{ { HostKey: h.hk, Success: isSuccessfulInteraction(err), diff --git a/worker/host_test.go b/worker/host_test.go index a993c12e1..3d124e9aa 100644 --- a/worker/host_test.go +++ b/worker/host_test.go @@ -13,7 +13,6 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/internal/test" "lukechampine.com/frand" ) @@ -22,7 +21,7 @@ type ( testHost struct { *hostMock *contractMock - hptFn func() hostdb.HostPriceTable + hptFn func() api.HostPriceTable } testHostManager struct { @@ -57,7 +56,7 @@ func newTestHost(h *hostMock, c *contractMock) *testHost { return newTestHostCustom(h, c, newTestHostPriceTable) } -func newTestHostCustom(h *hostMock, c *contractMock, hptFn func() hostdb.HostPriceTable) *testHost { +func newTestHostCustom(h *hostMock, c *contractMock, hptFn func() api.HostPriceTable) *testHost { return &testHost{ hostMock: h, contractMock: c, @@ -65,11 +64,11 @@ func newTestHostCustom(h *hostMock, c *contractMock, hptFn func() hostdb.HostPri } } -func newTestHostPriceTable() hostdb.HostPriceTable { +func newTestHostPriceTable() api.HostPriceTable { var uid rhpv3.SettingsID frand.Read(uid[:]) - return hostdb.HostPriceTable{ + return api.HostPriceTable{ HostPriceTable: rhpv3.HostPriceTable{UID: uid, HostBlockHeight: 100, Validity: time.Minute}, Expiry: time.Now().Add(time.Minute), } @@ -103,7 +102,7 @@ func (h *testHost) FetchRevision(ctx context.Context, fetchTimeout time.Duration return rev, nil } -func (h *testHost) FetchPriceTable(ctx context.Context, rev *types.FileContractRevision) (hostdb.HostPriceTable, error) { +func (h *testHost) FetchPriceTable(ctx context.Context, rev *types.FileContractRevision) (api.HostPriceTable, error) { return h.hptFn(), nil } diff --git a/worker/interactions.go b/worker/interactions.go index 2107ae582..34e47953a 100644 --- a/worker/interactions.go +++ b/worker/interactions.go @@ -1,13 +1,13 @@ package worker import ( - "go.sia.tech/renterd/hostdb" + "go.sia.tech/renterd/api" ) type ( HostInteractionRecorder interface { - RecordHostScan(...hostdb.HostScan) - RecordPriceTableUpdate(...hostdb.PriceTableUpdate) + RecordHostScan(...api.HostScan) + RecordPriceTableUpdate(...api.HostPriceTableUpdate) } ) diff --git a/worker/mocks_test.go b/worker/mocks_test.go index baf83b39d..c24b67df3 100644 --- a/worker/mocks_test.go +++ b/worker/mocks_test.go @@ -15,7 +15,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/object" "go.sia.tech/renterd/webhooks" ) @@ -267,10 +266,8 @@ func newHostMock(hk types.PublicKey) *hostMock { return &hostMock{ hk: hk, hi: api.Host{ - Host: hostdb.Host{ - PublicKey: hk, - Scanned: true, - }, + PublicKey: hk, + Scanned: true, }, } } @@ -298,11 +295,11 @@ func (hs *hostStoreMock) Host(ctx context.Context, hostKey types.PublicKey) (api return h.hi, nil } -func (hs *hostStoreMock) RecordHostScans(ctx context.Context, scans []hostdb.HostScan) error { +func (hs *hostStoreMock) RecordHostScans(ctx context.Context, scans []api.HostScan) error { return nil } -func (hs *hostStoreMock) RecordPriceTables(ctx context.Context, priceTableUpdate []hostdb.PriceTableUpdate) error { +func (hs *hostStoreMock) RecordPriceTables(ctx context.Context, priceTableUpdate []api.HostPriceTableUpdate) error { return nil } diff --git a/worker/pricetables.go b/worker/pricetables.go index 1bc2ee009..9ca4b1541 100644 --- a/worker/pricetables.go +++ b/worker/pricetables.go @@ -10,7 +10,7 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" - "go.sia.tech/renterd/hostdb" + "go.sia.tech/renterd/api" "lukechampine.com/frand" ) @@ -47,14 +47,14 @@ type ( hk types.PublicKey mu sync.Mutex - hpt hostdb.HostPriceTable + hpt api.HostPriceTable update *priceTableUpdate } priceTableUpdate struct { err error done chan struct{} - hpt hostdb.HostPriceTable + hpt api.HostPriceTable } ) @@ -75,7 +75,7 @@ func newPriceTables(hm HostManager, hs HostStore) *priceTables { } // fetch returns a price table for the given host -func (pts *priceTables) fetch(ctx context.Context, hk types.PublicKey, rev *types.FileContractRevision) (hostdb.HostPriceTable, error) { +func (pts *priceTables) fetch(ctx context.Context, hk types.PublicKey, rev *types.FileContractRevision) (api.HostPriceTable, error) { pts.mu.Lock() pt, exists := pts.priceTables[hk] if !exists { @@ -105,7 +105,7 @@ func (pt *priceTable) ongoingUpdate() (bool, *priceTableUpdate) { return ongoing, pt.update } -func (p *priceTable) fetch(ctx context.Context, rev *types.FileContractRevision) (hpt hostdb.HostPriceTable, err error) { +func (p *priceTable) fetch(ctx context.Context, rev *types.FileContractRevision) (hpt api.HostPriceTable, err error) { // grab the current price table p.mu.Lock() hpt = p.hpt @@ -115,7 +115,7 @@ func (p *priceTable) fetch(ctx context.Context, rev *types.FileContractRevision) // current price table is considered to gouge on the block height gc, err := GougingCheckerFromContext(ctx, false) if err != nil { - return hostdb.HostPriceTable{}, err + return api.HostPriceTable{}, err } // figure out whether we should update the price table, if not we can return @@ -137,7 +137,7 @@ func (p *priceTable) fetch(ctx context.Context, rev *types.FileContractRevision) } else if ongoing { select { case <-ctx.Done(): - return hostdb.HostPriceTable{}, fmt.Errorf("%w; %w", errPriceTableUpdateTimedOut, ctx.Err()) + return api.HostPriceTable{}, fmt.Errorf("%w; %w", errPriceTableUpdateTimedOut, ctx.Err()) case <-update.done: } return update.hpt, update.err @@ -166,14 +166,14 @@ func (p *priceTable) fetch(ctx context.Context, rev *types.FileContractRevision) // sanity check the host has been scanned before fetching the price table if !host.Scanned { - return hostdb.HostPriceTable{}, fmt.Errorf("host %v was not scanned", p.hk) + return api.HostPriceTable{}, fmt.Errorf("host %v was not scanned", p.hk) } // otherwise fetch it h := p.hm.Host(p.hk, types.FileContractID{}, host.Settings.SiamuxAddr()) hpt, err = h.FetchPriceTable(ctx, rev) if err != nil { - return hostdb.HostPriceTable{}, fmt.Errorf("failed to update pricetable, err %v", err) + return api.HostPriceTable{}, fmt.Errorf("failed to update pricetable, err %v", err) } return diff --git a/worker/pricetables_test.go b/worker/pricetables_test.go index 55b0f7057..22c021ccb 100644 --- a/worker/pricetables_test.go +++ b/worker/pricetables_test.go @@ -7,7 +7,6 @@ import ( "time" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" ) func TestPriceTables(t *testing.T) { @@ -45,7 +44,7 @@ func TestPriceTables(t *testing.T) { // manage the host, make sure fetching the price table blocks fetchPTBlockChan := make(chan struct{}) validPT := newTestHostPriceTable() - hm.addHost(newTestHostCustom(h, c, func() hostdb.HostPriceTable { + hm.addHost(newTestHostCustom(h, c, func() api.HostPriceTable { <-fetchPTBlockChan return validPT })) diff --git a/worker/rhpv3.go b/worker/rhpv3.go index c0404b128..22b75adc3 100644 --- a/worker/rhpv3.go +++ b/worker/rhpv3.go @@ -18,7 +18,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/mux/v1" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/internal/utils" "go.sia.tech/siad/crypto" "go.uber.org/zap" @@ -623,36 +622,36 @@ func processPayment(s *streamV3, payment rhpv3.PaymentMethod) error { type PriceTablePaymentFunc func(pt rhpv3.HostPriceTable) (rhpv3.PaymentMethod, error) // RPCPriceTable calls the UpdatePriceTable RPC. -func RPCPriceTable(ctx context.Context, t *transportV3, paymentFunc PriceTablePaymentFunc) (_ hostdb.HostPriceTable, err error) { +func RPCPriceTable(ctx context.Context, t *transportV3, paymentFunc PriceTablePaymentFunc) (_ api.HostPriceTable, err error) { defer wrapErr(&err, "PriceTable") s, err := t.DialStream(ctx) if err != nil { - return hostdb.HostPriceTable{}, err + return api.HostPriceTable{}, err } defer s.Close() var pt rhpv3.HostPriceTable var ptr rhpv3.RPCUpdatePriceTableResponse if err := s.WriteRequest(rhpv3.RPCUpdatePriceTableID, nil); err != nil { - return hostdb.HostPriceTable{}, fmt.Errorf("couldn't send RPCUpdatePriceTableID: %w", err) + return api.HostPriceTable{}, fmt.Errorf("couldn't send RPCUpdatePriceTableID: %w", err) } else if err := s.ReadResponse(&ptr, maxPriceTableSize); err != nil { - return hostdb.HostPriceTable{}, fmt.Errorf("couldn't read RPCUpdatePriceTableResponse: %w", err) + return api.HostPriceTable{}, fmt.Errorf("couldn't read RPCUpdatePriceTableResponse: %w", err) } else if err := json.Unmarshal(ptr.PriceTableJSON, &pt); err != nil { - return hostdb.HostPriceTable{}, fmt.Errorf("couldn't unmarshal price table: %w", err) + return api.HostPriceTable{}, fmt.Errorf("couldn't unmarshal price table: %w", err) } else if payment, err := paymentFunc(pt); err != nil { - return hostdb.HostPriceTable{}, fmt.Errorf("couldn't create payment: %w", err) + return api.HostPriceTable{}, fmt.Errorf("couldn't create payment: %w", err) } else if payment == nil { - return hostdb.HostPriceTable{ + return api.HostPriceTable{ HostPriceTable: pt, Expiry: time.Now(), }, nil // intended not to pay } else if err := processPayment(s, payment); err != nil { - return hostdb.HostPriceTable{}, fmt.Errorf("couldn't process payment: %w", err) + return api.HostPriceTable{}, fmt.Errorf("couldn't process payment: %w", err) } else if err := s.ReadResponse(&rhpv3.RPCPriceTableResponse{}, 0); err != nil { - return hostdb.HostPriceTable{}, fmt.Errorf("couldn't read RPCPriceTableResponse: %w", err) + return api.HostPriceTable{}, fmt.Errorf("couldn't read RPCPriceTableResponse: %w", err) } else { - return hostdb.HostPriceTable{ + return api.HostPriceTable{ HostPriceTable: pt, Expiry: time.Now().Add(pt.Validity), }, nil diff --git a/worker/worker.go b/worker/worker.go index d0de33f71..c78be49ea 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -24,7 +24,6 @@ import ( "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" "go.sia.tech/renterd/build" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/object" "go.sia.tech/renterd/webhooks" @@ -106,8 +105,8 @@ type ( } HostStore interface { - RecordHostScans(ctx context.Context, scans []hostdb.HostScan) error - RecordPriceTables(ctx context.Context, priceTableUpdate []hostdb.PriceTableUpdate) error + RecordHostScans(ctx context.Context, scans []api.HostScan) error + RecordPriceTables(ctx context.Context, priceTableUpdate []api.HostPriceTableUpdate) error RecordContractSpending(ctx context.Context, records []api.ContractSpendingRecord) error Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) @@ -355,9 +354,9 @@ func (w *worker) rhpPriceTableHandler(jc jape.Context) { // defer interaction recording var err error - var hpt hostdb.HostPriceTable + var hpt api.HostPriceTable defer func() { - w.bus.RecordPriceTables(ctx, []hostdb.PriceTableUpdate{ + w.bus.RecordPriceTables(ctx, []api.HostPriceTableUpdate{ { HostKey: rptr.HostKey, Success: isSuccessfulInteraction(err), @@ -1524,7 +1523,7 @@ func (w *worker) scanHost(ctx context.Context, timeout time.Duration, hostKey ty // record scans that timed out. recordCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - scanErr := w.bus.RecordHostScans(recordCtx, []hostdb.HostScan{ + scanErr := w.bus.RecordHostScans(recordCtx, []api.HostScan{ { HostKey: hostKey, Success: isSuccessfulInteraction(err), From 419b21e8bead113f2e900760dfd5bc44123a6a35 Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 25 Mar 2024 21:23:47 +0100 Subject: [PATCH 20/23] stores: update SearchHosts --- autopilot/contractor.go | 2 +- stores/hostdb.go | 51 +++++++++++++---------------------------- stores/hostdb_test.go | 24 ++++++++----------- 3 files changed, 27 insertions(+), 50 deletions(-) diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 0e85e4302..2be7fe9e9 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -720,7 +720,7 @@ func (c *contractor) runContractChecks(ctx context.Context, contracts []api.Cont } // fetch host checks - check, ok := host.Checks[c.ap.id] + check, ok := hostChecks[hk] if !ok { // this is only possible due to developer error, if there is no // check the host would have been missing, so we treat it the same diff --git a/stores/hostdb.go b/stores/hostdb.go index 822e97f95..ee0277624 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -487,21 +487,14 @@ func (e *dbBlocklistEntry) blocks(h dbHost) bool { // Host returns information about a host. func (ss *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) { - var h dbHost - - tx := ss.db. - WithContext(ctx). - Where(&dbHost{PublicKey: publicKey(hostKey)}). - Preload("Allowlist"). - Preload("Blocklist"). - Take(&h) - if errors.Is(tx.Error, gorm.ErrRecordNotFound) { + hosts, err := ss.SearchHosts(ctx, "", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", []types.PublicKey{hostKey}, 0, 1) + if err != nil { + return api.Host{}, err + } else if len(hosts) == 0 { return api.Host{}, api.ErrHostNotFound - } else if tx.Error != nil { - return api.Host{}, tx.Error + } else { + return hosts[0], nil } - - return h.convert(ss.isBlocked(h)), nil } func (ss *SQLStore) UpdateHostCheck(ctx context.Context, autopilotID string, hk types.PublicKey, hc api.HostCheck) (err error) { @@ -610,10 +603,6 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, autopilotID, filterMode, us return nil, ErrNegativeOffset } - // TODO PJ: use - _ = autopilotID - _ = usabilityMode - // validate filterMode switch filterMode { case api.HostFilterModeAllowed: @@ -631,7 +620,7 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, autopilotID, filterMode, us hostFilter(filterMode, ss.hasAllowlist(), ss.hasBlocklist()), hostNetAddress(addressContains), hostPublicKey(keyIn), - usabilityFilter(usabilityMode), + usabilityFilter(autopilotID, usabilityMode), ) // preload allowlist and blocklist @@ -641,23 +630,9 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, autopilotID, filterMode, us Preload("Blocklist") } - // filter checks - if autopilotID != "" { - query = query.Preload("Checks.DBAutopilot", "identifier = ?", autopilotID) - } else { - query = query.Preload("Checks.DBAutopilot") - } - // query = query. - // Preload("Checks.DBAutopilot"). - // Scopes( - // autopilotFilter(autopilotID), - // usabilityFilter(usabilityMode), - // ) - var hosts []api.Host var fullHosts []dbHost err := query. - Debug(). Offset(offset). Limit(limit). FindInBatches(&fullHosts, hostRetrievalBatchSize, func(tx *gorm.DB, batch int) error { @@ -1144,13 +1119,19 @@ func hostFilter(filterMode string, hasAllowlist, hasBlocklist bool) func(*gorm.D } } -func usabilityFilter(usabilityMode string) func(*gorm.DB) *gorm.DB { +func usabilityFilter(autopilotID, usabilityMode string) func(*gorm.DB) *gorm.DB { return func(db *gorm.DB) *gorm.DB { switch usabilityMode { case api.UsabilityFilterModeUsable: - db = db.Preload("Checks", "usability_blocked = ? AND usability_offline = ? AND usability_low_score = ? AND usability_redundant_ip = ? AND usability_gouging = ? AND usability_not_accepting_contracts = ? AND usability_not_announced = ? AND usability_not_completing_scan = ?", false, false, false, false, false, false, false, false) + db = db. + Joins("INNER JOIN host_checks hc on hc.db_host_id = hosts.id"). + Joins("INNER JOIN autopilots a on a.id = hc.db_autopilot_id AND a.identifier = ?", autopilotID). + Where("hc.usability_blocked = ? AND hc.usability_offline = ? AND hc.usability_low_score = ? AND hc.usability_redundant_ip = ? AND hc.usability_gouging = ? AND hc.usability_not_accepting_contracts = ? AND hc.usability_not_announced = ? AND hc.usability_not_completing_scan = ?", false, false, false, false, false, false, false, false) case api.UsabilityFilterModeUnusable: - db = db.Preload("Checks", "usability_blocked = ? OR usability_offline = ? OR usability_low_score = ? OR usability_redundant_ip = ? OR usability_gouging = ? OR usability_not_accepting_contracts = ? OR usability_not_announced = ? OR usability_not_completing_scan = ?", true, true, true, true, true, true, true, true) + db = db. + Joins("INNER JOIN host_checks hc on hc.db_host_id = hosts.id"). + Joins("INNER JOIN autopilots a on a.id = hc.db_autopilot_id AND a.identifier = ?", autopilotID). + Where("hc.usability_blocked = ? OR hc.usability_offline = ? OR hc.usability_low_score = ? OR hc.usability_redundant_ip = ? OR hc.usability_gouging = ? OR hc.usability_not_accepting_contracts = ? OR hc.usability_not_announced = ? OR hc.usability_not_completing_scan = ?", true, true, true, true, true, true, true, true) case api.UsabilityFilterModeAll: // do nothing } diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index 196170b01..283468474 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -408,32 +408,28 @@ func TestSearchHosts(t *testing.T) { his, err = ss.SearchHosts(context.Background(), ap1, api.HostFilterModeAll, api.UsabilityFilterModeUsable, "", nil, 0, -1) if err != nil { t.Fatal(err) - } else if cnt != 3 { - t.Fatal("unexpected", cnt) + } else if len(his) != 1 { + t.Fatal("unexpected", len(his)) } - // assert h1 and h2 have the expected checks + // assert h1 has the expected checks if c1, ok := his[0].Checks[ap1]; !ok || c1 != h1c { t.Fatal("unexpected", c1, ok) - } else if _, ok := his[1].Checks[ap1]; ok { - t.Fatal("unexpected", ok) - } else if _, ok := his[1].Checks[ap2]; ok { - t.Fatal("unexpected") } his, err = ss.SearchHosts(context.Background(), ap1, api.HostFilterModeAll, api.UsabilityFilterModeUnusable, "", nil, 0, -1) if err != nil { t.Fatal(err) - } else if cnt != 3 { - t.Fatal("unexpected", cnt) + } else if len(his) != 1 { + t.Fatal("unexpected", len(his)) + } else if his[0].Host.PublicKey != hk2 { + t.Fatal("unexpected") } - // assert h1 and h2 have the expected checks - if _, ok := his[0].Checks[ap1]; ok { + // assert only ap1 check is there + if _, ok := his[0].Checks[ap1]; !ok { t.Fatal("unexpected") - } else if c2, ok := his[1].Checks[ap1]; !ok || c2 != h2c1 { - t.Fatal("unexpected", ok) - } else if _, ok := his[1].Checks[ap2]; ok { + } else if _, ok := his[0].Checks[ap2]; ok { t.Fatal("unexpected") } From c68a88cde9079eaa746b695046e53270b297df38 Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 25 Mar 2024 21:51:42 +0100 Subject: [PATCH 21/23] lint: fix nesting --- stores/hostdb_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index bffca5c0f..6adf19968 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -422,7 +422,7 @@ func TestSearchHosts(t *testing.T) { t.Fatal(err) } else if len(his) != 1 { t.Fatal("unexpected", len(his)) - } else if his[0].Host.PublicKey != hk2 { + } else if his[0].PublicKey != hk2 { t.Fatal("unexpected") } From 59d6009dc60b0bfae8e5d01ca77731d832fba3e2 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 26 Mar 2024 13:36:46 +0100 Subject: [PATCH 22/23] contractor: remove error check --- autopilot/contractor.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 2be7fe9e9..bd8235511 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -286,9 +286,6 @@ func (c *contractor) performContractMaintenance(ctx context.Context, w Worker) ( // run contract checks updatedSet, toArchive, toStopUsing, toRefresh, toRenew := c.runContractChecks(ctx, contracts, isInCurrentSet, checks, cs.BlockHeight) - if err != nil { - return false, fmt.Errorf("failed to run contract checks, err: %v", err) - } // update host checks for hk, check := range checks { From 2f42d58ed8cb37fe36f8c20ebbeab0e7b67cde97 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 27 Mar 2024 16:42:38 +0100 Subject: [PATCH 23/23] api: omit unusablereasons if empty --- api/host.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/host.go b/api/host.go index c1e72aa16..030f2f3cf 100644 --- a/api/host.go +++ b/api/host.go @@ -80,7 +80,7 @@ type ( Score float64 `json:"score"` ScoreBreakdown HostScoreBreakdown `json:"scoreBreakdown"` Usable bool `json:"usable"` - UnusableReasons []string `json:"unusableReasons"` + UnusableReasons []string `json:"unusableReasons,omitempty"` } )