From dc56acaab730daa70c10547d5a31f750b4187d32 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 20 Mar 2024 14:10:25 +0100 Subject: [PATCH 01/16] stores: add dbHostInfo --- api/autopilot.go | 59 ---- api/host.go | 82 +++++ stores/hostdb.go | 331 ++++++++++++++++-- stores/hostdb_test.go | 217 ++++++++++++ stores/migrations.go | 6 + .../mysql/main/migration_00007_host_info.sql | 52 +++ stores/migrations/mysql/main/schema.sql | 53 +++ .../sqlite/main/migration_00007_host_info.sql | 52 +++ stores/migrations/sqlite/main/schema.sql | 19 + 9 files changed, 787 insertions(+), 84 deletions(-) create mode 100644 stores/migrations/mysql/main/migration_00007_host_info.sql create mode 100644 stores/migrations/sqlite/main/migration_00007_host_info.sql diff --git a/api/autopilot.go b/api/autopilot.go index fdd6c4942..22598b28c 100644 --- a/api/autopilot.go +++ b/api/autopilot.go @@ -2,8 +2,6 @@ package api import ( "errors" - "fmt" - "strings" "go.sia.tech/core/types" "go.sia.tech/renterd/hostdb" @@ -136,65 +134,8 @@ type ( Usable bool `json:"usable"` UnusableReasons []string `json:"unusableReasons"` } - - HostGougingBreakdown struct { - ContractErr string `json:"contractErr"` - DownloadErr string `json:"downloadErr"` - GougingErr string `json:"gougingErr"` - PruneErr string `json:"pruneErr"` - UploadErr string `json:"uploadErr"` - } - - HostScoreBreakdown struct { - Age float64 `json:"age"` - Collateral float64 `json:"collateral"` - Interactions float64 `json:"interactions"` - StorageRemaining float64 `json:"storageRemaining"` - Uptime float64 `json:"uptime"` - Version float64 `json:"version"` - Prices float64 `json:"prices"` - } ) -func (sb HostScoreBreakdown) String() string { - return fmt.Sprintf("Age: %v, Col: %v, Int: %v, SR: %v, UT: %v, V: %v, Pr: %v", sb.Age, sb.Collateral, sb.Interactions, sb.StorageRemaining, sb.Uptime, sb.Version, sb.Prices) -} - -func (hgb HostGougingBreakdown) Gouging() bool { - for _, err := range []string{ - hgb.ContractErr, - hgb.DownloadErr, - hgb.GougingErr, - hgb.PruneErr, - hgb.UploadErr, - } { - if err != "" { - return true - } - } - return false -} - -func (hgb HostGougingBreakdown) String() string { - var reasons []string - for _, errStr := range []string{ - hgb.ContractErr, - hgb.DownloadErr, - hgb.GougingErr, - hgb.PruneErr, - hgb.UploadErr, - } { - if errStr != "" { - reasons = append(reasons, errStr) - } - } - return strings.Join(reasons, ";") -} - -func (sb HostScoreBreakdown) Score() float64 { - return sb.Age * sb.Collateral * sb.Interactions * sb.StorageRemaining * sb.Uptime * sb.Version * sb.Prices -} - func (c AutopilotConfig) Validate() error { if c.Hosts.MaxDowntimeHours > 99*365*24 { return ErrMaxDowntimeHoursTooHigh diff --git a/api/host.go b/api/host.go index 293403c0a..2c1e82f30 100644 --- a/api/host.go +++ b/api/host.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net/url" + "strings" "go.sia.tech/core/types" "go.sia.tech/renterd/hostdb" @@ -23,6 +24,10 @@ var ( // ErrHostNotFound is returned when a host can't be retrieved from the // database. ErrHostNotFound = errors.New("host doesn't exist in hostdb") + + // ErrHostInfoNotFound is returned when host info can't be retrieved from + // the database. + ErrHostInfoNotFound = errors.New("host info doesn't exist in hostdb") ) type ( @@ -109,3 +114,80 @@ func (opts HostsForScanningOptions) Apply(values url.Values) { values.Set("lastScan", TimeRFC3339(opts.MaxLastScan).String()) } } + +type ( + HostInfo struct { + Host hostdb.Host `json:"host"` + Gouging HostGougingBreakdown `json:"gouging"` + Score HostScoreBreakdown `json:"score"` + Usability HostUsabilityBreakdown `json:"usability"` + } + + HostGougingBreakdown struct { + ContractErr string `json:"contractErr"` + DownloadErr string `json:"downloadErr"` + GougingErr string `json:"gougingErr"` + PruneErr string `json:"pruneErr"` + UploadErr string `json:"uploadErr"` + } + + HostScoreBreakdown struct { + Age float64 `json:"age"` + Collateral float64 `json:"collateral"` + Interactions float64 `json:"interactions"` + StorageRemaining float64 `json:"storageRemaining"` + Uptime float64 `json:"uptime"` + Version float64 `json:"version"` + Prices float64 `json:"prices"` + } + + HostUsabilityBreakdown struct { + Blocked bool `json:"blocked"` + Offline bool `json:"offline"` + LowScore bool `json:"lowScore"` + RedundantIP bool `json:"redundantIP"` + Gouging bool `json:"gouging"` + NotAcceptingContracts bool `json:"notAcceptingContracts"` + NotAnnounced bool `json:"notAnnounced"` + NotCompletingScan bool `json:"notCompletingScan"` + } +) + +func (sb HostScoreBreakdown) String() string { + return fmt.Sprintf("Age: %v, Col: %v, Int: %v, SR: %v, UT: %v, V: %v, Pr: %v", sb.Age, sb.Collateral, sb.Interactions, sb.StorageRemaining, sb.Uptime, sb.Version, sb.Prices) +} + +func (hgb HostGougingBreakdown) Gouging() bool { + for _, err := range []string{ + hgb.ContractErr, + hgb.DownloadErr, + hgb.GougingErr, + hgb.PruneErr, + hgb.UploadErr, + } { + if err != "" { + return true + } + } + return false +} + +func (hgb HostGougingBreakdown) String() string { + var reasons []string + for _, errStr := range []string{ + hgb.ContractErr, + hgb.DownloadErr, + hgb.GougingErr, + hgb.PruneErr, + hgb.UploadErr, + } { + if errStr != "" { + reasons = append(reasons, errStr) + } + } + return strings.Join(reasons, ";") +} + +func (sb HostScoreBreakdown) Score() float64 { + return sb.Age * sb.Collateral * sb.Interactions * sb.StorageRemaining * sb.Uptime * sb.Version * sb.Prices +} diff --git a/stores/hostdb.go b/stores/hostdb.go index 95e37a26c..c344fd83f 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -80,6 +80,44 @@ type ( Blocklist []dbBlocklistEntry `gorm:"many2many:host_blocklist_entry_hosts;constraint:OnDelete:CASCADE"` } + // dbHostInfo contains information about a host that is collected and used + // by the autopilot. + dbHostInfo struct { + Model + + DBAutopilotID uint `gorm:"index:idx_host_infos_id,unique"` + DBAutopilot dbAutopilot + + DBHostID uint `gorm:"index:idx_host_infos_id,unique"` + DBHost dbHost + + // usability + UsabilityBlocked bool `gorm:"index:idx_host_infos_usability_blocked"` + UsabilityOffline bool `gorm:"index:idx_host_infos_usability_offline"` + UsabilityLowScore bool `gorm:"index:idx_host_infos_usability_low_score"` + UsabilityRedundantIP bool `gorm:"index:idx_host_infos_usability_redundant_ip"` + UsabilityGouging bool `gorm:"index:idx_host_infos_usability_gouging"` + UsabilityNotAcceptingContracts bool `gorm:"index:idx_host_infos_usability_not_accepting_contracts"` + UsabilityNotAnnounced bool `gorm:"index:idx_host_infos_usability_not_announced"` + UsabilityNotCompletingScan bool `gorm:"index:idx_host_infos_usability_not_completing_scan"` + + // score + ScoreAge float64 `gorm:"index:idx_host_infos_score_age"` + ScoreCollateral float64 `gorm:"index:idx_host_infos_score_collateral"` + ScoreInteractions float64 `gorm:"index:idx_host_infos_score_interactions"` + ScoreStorageRemaining float64 `gorm:"index:idx_host_infos_score_storage_remaining"` + ScoreUptime float64 `gorm:"index:idx_host_infos_score_uptime"` + ScoreVersion float64 `gorm:"index:idx_host_infos_score_version"` + ScorePrices float64 `gorm:"index:idx_host_infos_score_prices"` + + // gouging + GougingContractErr string + GougingDownloadErr string + GougingGougingErr string + GougingPruneErr string + GougingUploadErr string + } + // dbAllowlistEntry defines a table that stores the host blocklist. dbAllowlistEntry struct { Model @@ -263,6 +301,9 @@ func (dbConsensusInfo) TableName() string { return "consensus_infos" } // TableName implements the gorm.Tabler interface. func (dbHost) TableName() string { return "hosts" } +// TableName implements the gorm.Tabler interface. +func (dbHostInfo) TableName() string { return "host_infos" } + // TableName implements the gorm.Tabler interface. func (dbAllowlistEntry) TableName() string { return "host_allowlist_entries" } @@ -300,6 +341,68 @@ func (h dbHost) convert() hostdb.Host { } } +func (hi dbHostInfo) convert() api.HostInfo { + return api.HostInfo{ + Host: hi.DBHost.convert(), + Gouging: api.HostGougingBreakdown{ + ContractErr: hi.GougingContractErr, + DownloadErr: hi.GougingDownloadErr, + GougingErr: hi.GougingGougingErr, + PruneErr: hi.GougingPruneErr, + UploadErr: hi.GougingUploadErr, + }, + Score: api.HostScoreBreakdown{ + Age: hi.ScoreAge, + Collateral: hi.ScoreCollateral, + Interactions: hi.ScoreInteractions, + StorageRemaining: hi.ScoreStorageRemaining, + Uptime: hi.ScoreUptime, + Version: hi.ScoreVersion, + Prices: hi.ScorePrices, + }, + Usability: api.HostUsabilityBreakdown{ + Blocked: hi.UsabilityBlocked, + Offline: hi.UsabilityOffline, + LowScore: hi.UsabilityLowScore, + RedundantIP: hi.UsabilityRedundantIP, + Gouging: hi.UsabilityGouging, + NotAcceptingContracts: hi.UsabilityNotAcceptingContracts, + NotAnnounced: hi.UsabilityNotAnnounced, + NotCompletingScan: hi.UsabilityNotCompletingScan, + }, + } +} + +func convertHostInfo(apID, hID uint, gouging api.HostGougingBreakdown, score api.HostScoreBreakdown, usability api.HostUsabilityBreakdown) *dbHostInfo { + return &dbHostInfo{ + DBAutopilotID: apID, + DBHostID: hID, + + UsabilityBlocked: usability.Blocked, + UsabilityOffline: usability.Offline, + UsabilityLowScore: usability.LowScore, + UsabilityRedundantIP: usability.RedundantIP, + UsabilityGouging: usability.Gouging, + UsabilityNotAcceptingContracts: usability.NotAcceptingContracts, + UsabilityNotAnnounced: usability.NotAnnounced, + UsabilityNotCompletingScan: usability.NotCompletingScan, + + ScoreAge: score.Age, + ScoreCollateral: score.Collateral, + ScoreInteractions: score.Interactions, + ScoreStorageRemaining: score.StorageRemaining, + ScoreUptime: score.Uptime, + ScoreVersion: score.Version, + ScorePrices: score.Prices, + + GougingContractErr: gouging.ContractErr, + GougingDownloadErr: gouging.DownloadErr, + GougingGougingErr: gouging.GougingErr, + GougingPruneErr: gouging.PruneErr, + GougingUploadErr: gouging.UploadErr, + } +} + func (h *dbHost) BeforeCreate(tx *gorm.DB) (err error) { tx.Statement.AddClause(clause.OnConflict{ Columns: []clause.Column{{Name: "public_key"}}, @@ -426,6 +529,180 @@ func (ss *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (hostdb.H }, nil } +func (ss *SQLStore) HostInfo(ctx context.Context, autopilotID string, hk types.PublicKey) (hi api.HostInfo, err error) { + err = ss.db.Transaction(func(tx *gorm.DB) error { + // fetch ap id + var apID uint + if err := tx. + Model(&dbAutopilot{}). + Where("identifier = ?", autopilotID). + Select("id"). + Take(&apID). + Error; errors.Is(err, gorm.ErrRecordNotFound) { + return api.ErrAutopilotNotFound + } else if err != nil { + return err + } + + // fetch host id + var hID uint + if err := tx. + Model(&dbHost{}). + Where("public_key = ?", publicKey(hk)). + Select("id"). + Take(&hID). + Error; errors.Is(err, gorm.ErrRecordNotFound) { + return api.ErrHostNotFound + } else if err != nil { + return err + } + + // fetch host info + var entity dbHostInfo + if err := tx. + Model(&dbHostInfo{}). + Where("db_autopilot_id = ? AND db_host_id = ?", apID, hID). + Preload("DBHost"). + First(&entity). + Error; errors.Is(err, gorm.ErrRecordNotFound) { + return api.ErrHostInfoNotFound + } else if err != nil { + return err + } + + hi = entity.convert() + return nil + }) + return +} + +func (ss *SQLStore) HostInfos(ctx context.Context, autopilotID string, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) (his []api.HostInfo, err error) { + if offset < 0 { + return nil, ErrNegativeOffset + } + + err = ss.db.Transaction(func(tx *gorm.DB) error { + // fetch ap id + var apID uint + if err := tx. + Model(&dbAutopilot{}). + Where("identifier = ?", autopilotID). + Select("id"). + Take(&apID). + Error; errors.Is(err, gorm.ErrRecordNotFound) { + return api.ErrAutopilotNotFound + } else if err != nil { + return err + } + + // prepare query + query := tx. + Model(&dbHostInfo{}). + Where("db_autopilot_id = ?", apID). + Joins("DBHost") + + // apply mode filter + switch filterMode { + case api.HostFilterModeAllowed: + query = query.Scopes(ss.excludeBlocked("DBHost")) + case api.HostFilterModeBlocked: + query = query.Scopes(ss.excludeAllowed("DBHost")) + case api.HostFilterModeAll: + // nothing to do + default: + return fmt.Errorf("invalid filter mode: %v", filterMode) + } + + // apply usability filter + switch usabilityMode { + case api.UsabilityFilterModeUsable: + query = query.Where("usability_blocked = ? AND usability_offline = ? AND usability_low_score = ? AND usability_redundant_ip = ? AND usability_gouging = ? AND usability_not_accepting_contracts = ? AND usability_not_announced = ? AND usability_not_completing_scan = ?", + false, false, false, false, false, false, false, false) + case api.UsabilityFilterModeUnusable: + query = query.Where("usability_blocked = ? OR usability_offline = ? OR usability_low_score = ? OR usability_redundant_ip = ? OR usability_gouging = ? OR usability_not_accepting_contracts = ? OR usability_not_announced = ? OR usability_not_completing_scan = ?", + true, true, true, true, true, true, true, true) + case api.UsabilityFilterModeAll: + // nothing to do + default: + return fmt.Errorf("invalid usability mode: %v", usabilityMode) + } + + // apply address filter + if addressContains != "" { + query = query.Scopes(func(d *gorm.DB) *gorm.DB { + return d.Where("net_address LIKE ?", "%"+addressContains+"%") + }) + } + + // apply key filter + if len(keyIn) > 0 { + pubKeys := make([]publicKey, len(keyIn)) + for i, pk := range keyIn { + pubKeys[i] = publicKey(pk) + } + query = query.Scopes(func(d *gorm.DB) *gorm.DB { + return d.Where("public_key IN ?", pubKeys) + }) + } + + // fetch host info + var infos []dbHostInfo + if err := query. + Debug(). + Offset(offset). + Limit(limit). + Find(&infos). + Error; err != nil { + return err + } + for _, hi := range infos { + his = append(his, hi.convert()) + } + return nil + }) + return +} + +func (ss *SQLStore) UpdateHostInfo(ctx context.Context, autopilotID string, hk types.PublicKey, gouging api.HostGougingBreakdown, score api.HostScoreBreakdown, usability api.HostUsabilityBreakdown) (err error) { + err = ss.db.Transaction(func(tx *gorm.DB) error { + // fetch ap id + var apID uint + if err := tx. + Model(&dbAutopilot{}). + Where("identifier = ?", autopilotID). + Select("id"). + Take(&apID). + Error; errors.Is(err, gorm.ErrRecordNotFound) { + return api.ErrAutopilotNotFound + } else if err != nil { + return err + } + + // fetch host id + var hID uint + if err := tx. + Model(&dbHost{}). + Where("public_key = ?", publicKey(hk)). + Select("id"). + Take(&hID). + Error; errors.Is(err, gorm.ErrRecordNotFound) { + return api.ErrHostNotFound + } else if err != nil { + return err + } + + // update host info + return tx. + Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "db_autopilot_id"}, {Name: "db_host_id"}}, + UpdateAll: true, + }). + Create(convertHostInfo(apID, hID, gouging, score, usability)). + Error + }) + return +} + // HostsForScanning returns the address of hosts for scanning. func (ss *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]hostdb.HostAddress, error) { if offset < 0 { @@ -471,9 +748,9 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains query := ss.db switch filterMode { case api.HostFilterModeAllowed: - query = query.Scopes(ss.excludeBlocked) + query = query.Scopes(ss.excludeBlocked("hosts")) case api.HostFilterModeBlocked: - query = query.Scopes(ss.excludeAllowed) + query = query.Scopes(ss.excludeAllowed("hosts")) blocked = true case api.HostFilterModeAll: // preload allowlist and blocklist @@ -932,37 +1209,41 @@ func (ss *SQLStore) processConsensusChangeHostDB(cc modules.ConsensusChange) { // excludeBlocked can be used as a scope for a db transaction to exclude blocked // hosts. -func (ss *SQLStore) excludeBlocked(db *gorm.DB) *gorm.DB { - ss.mu.Lock() - defer ss.mu.Unlock() +func (ss *SQLStore) excludeBlocked(alias string) func(db *gorm.DB) *gorm.DB { + return func(db *gorm.DB) *gorm.DB { + ss.mu.Lock() + defer ss.mu.Unlock() - if ss.hasAllowlist { - db = db.Where("EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") - } - if ss.hasBlocklist { - db = db.Where("NOT EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") + if ss.hasAllowlist { + db = db.Where(fmt.Sprintf("EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", alias)) + } + if ss.hasBlocklist { + db = db.Where(fmt.Sprintf("NOT EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", alias)) + } + return db } - return db } // excludeAllowed can be used as a scope for a db transaction to exclude allowed // hosts. -func (ss *SQLStore) excludeAllowed(db *gorm.DB) *gorm.DB { - ss.mu.Lock() - defer ss.mu.Unlock() +func (ss *SQLStore) excludeAllowed(alias string) func(db *gorm.DB) *gorm.DB { + return func(db *gorm.DB) *gorm.DB { + ss.mu.Lock() + defer ss.mu.Unlock() - if ss.hasAllowlist { - db = db.Where("NOT EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") - } - if ss.hasBlocklist { - db = db.Where("EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") - } - if !ss.hasAllowlist && !ss.hasBlocklist { - // if neither an allowlist nor a blocklist exist, all hosts are allowed - // which means we return none - db = db.Where("1 = 0") + if ss.hasAllowlist { + db = db.Where(fmt.Sprintf("NOT EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", alias)) + } + if ss.hasBlocklist { + db = db.Where(fmt.Sprintf("EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", alias)) + } + if !ss.hasAllowlist && !ss.hasBlocklist { + // if neither an allowlist nor a blocklist exist, all hosts are allowed + // which means we return none + db = db.Where("1 = 0") + } + return db } - return db } func (ss *SQLStore) isBlocked(h dbHost) (blocked bool) { diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index 35872ea2d..8a75caf6d 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -1064,6 +1064,191 @@ func TestAnnouncementMaxAge(t *testing.T) { } } +func TestHostInfo(t *testing.T) { + ss := newTestSQLStore(t, defaultTestSQLStoreConfig) + defer ss.Close() + + // fetch info for a non-existing autopilot + _, err := ss.HostInfo(context.Background(), "foo", types.PublicKey{1}) + if !errors.Is(err, api.ErrAutopilotNotFound) { + t.Fatal(err) + } + + // add autopilot + err = ss.UpdateAutopilot(context.Background(), api.Autopilot{ID: "foo"}) + if err != nil { + t.Fatal(err) + } + + // fetch info for a non-existing host + _, err = ss.HostInfo(context.Background(), "foo", types.PublicKey{1}) + if !errors.Is(err, api.ErrHostNotFound) { + t.Fatal(err) + } + + // add host + err = ss.addTestHost(types.PublicKey{1}) + if err != nil { + t.Fatal(err) + } + h, err := ss.Host(context.Background(), types.PublicKey{1}) + if err != nil { + t.Fatal(err) + } + + // fetch non-existing info + _, err = ss.HostInfo(context.Background(), "foo", types.PublicKey{1}) + if !errors.Is(err, api.ErrHostInfoNotFound) { + t.Fatal(err) + } + + // add host info + want := newTestHostInfo(h.Host) + err = ss.UpdateHostInfo(context.Background(), "foo", types.PublicKey{1}, want.Gouging, want.Score, want.Usability) + if err != nil { + t.Fatal(err) + } + + // fetch info + got, err := ss.HostInfo(context.Background(), "foo", types.PublicKey{1}) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(got, want) { + t.Fatal("mismatch", cmp.Diff(got, want)) + } + + // update info + want.Score.Age = 0 + err = ss.UpdateHostInfo(context.Background(), "foo", types.PublicKey{1}, want.Gouging, want.Score, want.Usability) + if err != nil { + t.Fatal(err) + } + + // fetch info + got, err = ss.HostInfo(context.Background(), "foo", types.PublicKey{1}) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(got, want) { + t.Fatal("mismatch") + } + + // add another host info + err = ss.addCustomTestHost(types.PublicKey{2}, "bar.com:1000") + if err != nil { + t.Fatal(err) + } + err = ss.UpdateHostInfo(context.Background(), "foo", types.PublicKey{2}, want.Gouging, want.Score, want.Usability) + if err != nil { + t.Fatal(err) + } + + // fetch all infos for autopilot + his, err := ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 2 { + t.Fatal("unexpected") + } else if his[0].Host.PublicKey != (types.PublicKey{1}) || his[1].Host.PublicKey != (types.PublicKey{2}) { + t.Fatal("unexpected", his) + } + + // fetch infos using offset & limit + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 0, 1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 1, 1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 2, 1) + if err != nil { + t.Fatal(err) + } else if len(his) != 0 { + t.Fatal("unexpected") + } + + // fetch infos using net addresses + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "bar", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } else if his[0].Host.PublicKey != (types.PublicKey{2}) { + t.Fatal("unexpected", his) + } + + // fetch infos using keyIn + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", []types.PublicKey{{2}}, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } else if his[0].Host.PublicKey != (types.PublicKey{2}) { + t.Fatal("unexpected", his) + } + + // fetch infos using mode filters + err = ss.UpdateHostBlocklistEntries(context.Background(), []string{"bar.com:1000"}, nil, false) + if err != nil { + t.Fatal(err) + } + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAllowed, api.UsabilityFilterModeAll, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } else if his[0].Host.PublicKey != (types.PublicKey{1}) { + t.Fatal("unexpected", his) + } + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeBlocked, api.UsabilityFilterModeAll, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } else if his[0].Host.PublicKey != (types.PublicKey{2}) { + t.Fatal("unexpected", his) + } + err = ss.UpdateHostBlocklistEntries(context.Background(), nil, nil, true) + if err != nil { + t.Fatal(err) + } + + // fetch infos using usability filters + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeUsable, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 0 { + t.Fatal("unexpected") + } + + // update info + want.Usability.Blocked = false + want.Usability.Offline = false + want.Usability.LowScore = false + want.Usability.RedundantIP = false + want.Usability.Gouging = false + want.Usability.NotAcceptingContracts = false + want.Usability.NotAnnounced = false + want.Usability.NotCompletingScan = false + err = ss.UpdateHostInfo(context.Background(), "foo", types.PublicKey{1}, want.Gouging, want.Score, want.Usability) + if err != nil { + t.Fatal(err) + } + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeUsable, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } else if his[0].Host.PublicKey != (types.PublicKey{1}) { + t.Fatal("unexpected", his) + } +} + // addTestHosts adds 'n' hosts to the db and returns their keys. func (s *SQLStore) addTestHosts(n int) (keys []types.PublicKey, err error) { cnt, err := s.contractsCount() @@ -1156,3 +1341,35 @@ func newTestTransaction(ha modules.HostAnnouncement, sk types.PrivateKey) stypes buf.Write(encoding.Marshal(sk.SignHash(types.Hash256(crypto.HashObject(ha))))) return stypes.Transaction{ArbitraryData: [][]byte{buf.Bytes()}} } + +func newTestHostInfo(h hostdb.Host) api.HostInfo { + return api.HostInfo{ + Host: h, + Gouging: api.HostGougingBreakdown{ + ContractErr: "foo", + DownloadErr: "bar", + GougingErr: "baz", + PruneErr: "qux", + UploadErr: "quuz", + }, + Score: api.HostScoreBreakdown{ + Age: .1, + Collateral: .2, + Interactions: .3, + StorageRemaining: .4, + Uptime: .5, + Version: .6, + Prices: .7, + }, + Usability: api.HostUsabilityBreakdown{ + Blocked: true, + Offline: true, + LowScore: true, + RedundantIP: true, + Gouging: true, + NotAcceptingContracts: true, + NotAnnounced: true, + NotCompletingScan: true, + }, + } +} diff --git a/stores/migrations.go b/stores/migrations.go index 6ccc75964..9f874935a 100644 --- a/stores/migrations.go +++ b/stores/migrations.go @@ -62,6 +62,12 @@ func performMigrations(db *gorm.DB, logger *zap.SugaredLogger) error { return performMigration(tx, dbIdentifier, "00006_idx_objects_created_at", logger) }, }, + { + ID: "00007_host_info", + Migrate: func(tx *gorm.DB) error { + return performMigration(tx, dbIdentifier, "00007_host_info", logger) + }, + }, } // Create migrator. diff --git a/stores/migrations/mysql/main/migration_00007_host_info.sql b/stores/migrations/mysql/main/migration_00007_host_info.sql new file mode 100644 index 000000000..c13f5c396 --- /dev/null +++ b/stores/migrations/mysql/main/migration_00007_host_info.sql @@ -0,0 +1,52 @@ +-- dbHostInfo +CREATE TABLE `host_infos` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `created_at` datetime(3) DEFAULT NULL, + + `db_autopilot_id` bigint unsigned NOT NULL, + `db_host_id` bigint unsigned NOT NULL, + + `usability_blocked` boolean NOT NULL DEFAULT false, + `usability_offline` boolean NOT NULL DEFAULT false, + `usability_low_score` boolean NOT NULL DEFAULT false, + `usability_redundant_ip` boolean NOT NULL DEFAULT false, + `usability_gouging` boolean NOT NULL DEFAULT false, + `usability_not_accepting_contracts` boolean NOT NULL DEFAULT false, + `usability_not_announced` boolean NOT NULL DEFAULT false, + `usability_not_completing_scan` boolean NOT NULL DEFAULT false, + + `score_age` double NOT NULL, + `score_collateral` double NOT NULL, + `score_interactions` double NOT NULL, + `score_storage_remaining` double NOT NULL, + `score_uptime` double NOT NULL, + `score_version` double NOT NULL, + `score_prices` double NOT NULL, + + `gouging_contract_err` text, + `gouging_download_err` text, + `gouging_gouging_err` text, + `gouging_prune_err` text, + `gouging_upload_err` text, + + PRIMARY KEY (`id`), + UNIQUE KEY `idx_host_infos_id` (`db_autopilot_id`, `db_host_id`), + INDEX `idx_host_infos_usability_blocked` (`usability_blocked`), + INDEX `idx_host_infos_usability_offline` (`usability_offline`), + INDEX `idx_host_infos_usability_low_score` (`usability_low_score`), + INDEX `idx_host_infos_usability_redundant_ip` (`usability_redundant_ip`), + INDEX `idx_host_infos_usability_gouging` (`usability_gouging`), + INDEX `idx_host_infos_usability_not_accepting_contracts` (`usability_not_accepting_contracts`), + INDEX `idx_host_infos_usability_not_announced` (`usability_not_announced`), + INDEX `idx_host_infos_usability_not_completing_scan` (`usability_not_completing_scan`), + INDEX `idx_host_infos_score_age` (`score_age`), + INDEX `idx_host_infos_score_collateral` (`score_collateral`), + INDEX `idx_host_infos_score_interactions` (`score_interactions`), + INDEX `idx_host_infos_score_storage_remaining` (`score_storage_remaining`), + INDEX `idx_host_infos_score_uptime` (`score_uptime`), + INDEX `idx_host_infos_score_version` (`score_version`), + INDEX `idx_host_infos_score_prices` (`score_prices`), + + CONSTRAINT `fk_host_infos_autopilot` FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_host_infos_host` FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; diff --git a/stores/migrations/mysql/main/schema.sql b/stores/migrations/mysql/main/schema.sql index 68b42ae47..e39b7f963 100644 --- a/stores/migrations/mysql/main/schema.sql +++ b/stores/migrations/mysql/main/schema.sql @@ -422,5 +422,58 @@ CREATE TABLE `object_user_metadata` ( CONSTRAINT `fk_multipart_upload_user_metadata` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads` (`id`) ON DELETE SET NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +-- dbHostInfo +CREATE TABLE `host_infos` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `created_at` datetime(3) DEFAULT NULL, + + `db_autopilot_id` bigint unsigned NOT NULL, + `db_host_id` bigint unsigned NOT NULL, + + `usability_blocked` boolean NOT NULL DEFAULT false, + `usability_offline` boolean NOT NULL DEFAULT false, + `usability_low_score` boolean NOT NULL DEFAULT false, + `usability_redundant_ip` boolean NOT NULL DEFAULT false, + `usability_gouging` boolean NOT NULL DEFAULT false, + `usability_not_accepting_contracts` boolean NOT NULL DEFAULT false, + `usability_not_announced` boolean NOT NULL DEFAULT false, + `usability_not_completing_scan` boolean NOT NULL DEFAULT false, + + `score_age` double NOT NULL, + `score_collateral` double NOT NULL, + `score_interactions` double NOT NULL, + `score_storage_remaining` double NOT NULL, + `score_uptime` double NOT NULL, + `score_version` double NOT NULL, + `score_prices` double NOT NULL, + + `gouging_contract_err` text, + `gouging_download_err` text, + `gouging_gouging_err` text, + `gouging_prune_err` text, + `gouging_upload_err` text, + + PRIMARY KEY (`id`), + UNIQUE KEY `idx_host_infos_id` (`db_autopilot_id`, `db_host_id`), + INDEX `idx_host_infos_usability_blocked` (`usability_blocked`), + INDEX `idx_host_infos_usability_offline` (`usability_offline`), + INDEX `idx_host_infos_usability_low_score` (`usability_low_score`), + INDEX `idx_host_infos_usability_redundant_ip` (`usability_redundant_ip`), + INDEX `idx_host_infos_usability_gouging` (`usability_gouging`), + INDEX `idx_host_infos_usability_not_accepting_contracts` (`usability_not_accepting_contracts`), + INDEX `idx_host_infos_usability_not_announced` (`usability_not_announced`), + INDEX `idx_host_infos_usability_not_completing_scan` (`usability_not_completing_scan`), + INDEX `idx_host_infos_score_age` (`score_age`), + INDEX `idx_host_infos_score_collateral` (`score_collateral`), + INDEX `idx_host_infos_score_interactions` (`score_interactions`), + INDEX `idx_host_infos_score_storage_remaining` (`score_storage_remaining`), + INDEX `idx_host_infos_score_uptime` (`score_uptime`), + INDEX `idx_host_infos_score_version` (`score_version`), + INDEX `idx_host_infos_score_prices` (`score_prices`), + + CONSTRAINT `fk_host_infos_autopilot` FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_host_infos_host` FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + -- create default bucket INSERT INTO buckets (created_at, name) VALUES (CURRENT_TIMESTAMP, 'default'); \ No newline at end of file diff --git a/stores/migrations/sqlite/main/migration_00007_host_info.sql b/stores/migrations/sqlite/main/migration_00007_host_info.sql new file mode 100644 index 000000000..910dd637c --- /dev/null +++ b/stores/migrations/sqlite/main/migration_00007_host_info.sql @@ -0,0 +1,52 @@ +-- dbHostInfo +CREATE TABLE `host_infos` ( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `created_at` datetime, + + `db_autopilot_id` INTEGER NOT NULL, + `db_host_id` INTEGER NOT NULL, + + `usability_blocked` INTEGER NOT NULL DEFAULT 0, + `usability_offline` INTEGER NOT NULL DEFAULT 0, + `usability_low_score` INTEGER NOT NULL DEFAULT 0, + `usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, + `usability_gouging` INTEGER NOT NULL DEFAULT 0, + `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, + `usability_not_announced` INTEGER NOT NULL DEFAULT 0, + `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, + + `score_age` REAL NOT NULL, + `score_collateral` REAL NOT NULL, + `score_interactions` REAL NOT NULL, + `score_storage_remaining` REAL NOT NULL, + `score_uptime` REAL NOT NULL, + `score_version` REAL NOT NULL, + `score_prices` REAL NOT NULL, + + `gouging_contract_err` TEXT, + `gouging_download_err` TEXT, + `gouging_gouging_err` TEXT, + `gouging_prune_err` TEXT, + `gouging_upload_err` TEXT, + + FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, + FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE +); + +-- Indexes creation +CREATE UNIQUE INDEX `idx_host_infos_id` ON `host_infos` (`db_autopilot_id`, `db_host_id`); +CREATE INDEX `idx_host_infos_usability_blocked` ON `host_infos` (`usability_blocked`); +CREATE INDEX `idx_host_infos_usability_offline` ON `host_infos` (`usability_offline`); +CREATE INDEX `idx_host_infos_usability_low_score` ON `host_infos` (`usability_low_score`); +CREATE INDEX `idx_host_infos_usability_redundant_ip` ON `host_infos` (`usability_redundant_ip`); +CREATE INDEX `idx_host_infos_usability_gouging` ON `host_infos` (`usability_gouging`); +CREATE INDEX `idx_host_infos_usability_not_accepting_contracts` ON `host_infos` (`usability_not_accepting_contracts`); +CREATE INDEX `idx_host_infos_usability_not_announced` ON `host_infos` (`usability_not_announced`); +CREATE INDEX `idx_host_infos_usability_not_completing_scan` ON `host_infos` (`usability_not_completing_scan`); +CREATE INDEX `idx_host_infos_score_age` ON `host_infos` (`score_age`); +CREATE INDEX `idx_host_infos_score_collateral` ON `host_infos` (`score_collateral`); +CREATE INDEX `idx_host_infos_score_interactions` ON `host_infos` (`score_interactions`); +CREATE INDEX `idx_host_infos_score_storage_remaining` ON `host_infos` (`score_storage_remaining`); +CREATE INDEX `idx_host_infos_score_uptime` ON `host_infos` (`score_uptime`); +CREATE INDEX `idx_host_infos_score_version` ON `host_infos` (`score_version`); +CREATE INDEX `idx_host_infos_score_prices` ON `host_infos` (`score_prices`); diff --git a/stores/migrations/sqlite/main/schema.sql b/stores/migrations/sqlite/main/schema.sql index 9875e81e3..791fce1ca 100644 --- a/stores/migrations/sqlite/main/schema.sql +++ b/stores/migrations/sqlite/main/schema.sql @@ -149,5 +149,24 @@ CREATE UNIQUE INDEX `idx_module_event_url` ON `webhooks`(`module`,`event`,`url`) CREATE TABLE `object_user_metadata` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`db_object_id` integer DEFAULT NULL,`db_multipart_upload_id` integer DEFAULT NULL,`key` text NOT NULL,`value` text, CONSTRAINT `fk_object_user_metadata` FOREIGN KEY (`db_object_id`) REFERENCES `objects` (`id`) ON DELETE CASCADE, CONSTRAINT `fk_multipart_upload_user_metadata` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads` (`id`) ON DELETE SET NULL); CREATE UNIQUE INDEX `idx_object_user_metadata_key` ON `object_user_metadata`(`db_object_id`,`db_multipart_upload_id`,`key`); +-- dbHostInfo +CREATE TABLE `host_infos` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `created_at` datetime, `db_autopilot_id` INTEGER NOT NULL, `db_host_id` INTEGER NOT NULL, `usability_blocked` INTEGER NOT NULL DEFAULT 0, `usability_offline` INTEGER NOT NULL DEFAULT 0, `usability_low_score` INTEGER NOT NULL DEFAULT 0, `usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, `usability_gouging` INTEGER NOT NULL DEFAULT 0, `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, `usability_not_announced` INTEGER NOT NULL DEFAULT 0, `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, `score_age` REAL NOT NULL, `score_collateral` REAL NOT NULL, `score_interactions` REAL NOT NULL, `score_storage_remaining` REAL NOT NULL, `score_uptime` REAL NOT NULL, `score_version` REAL NOT NULL, `score_prices` REAL NOT NULL, `gouging_contract_err` TEXT, `gouging_download_err` TEXT, `gouging_gouging_err` TEXT, `gouging_prune_err` TEXT, `gouging_upload_err` TEXT, FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE); +CREATE UNIQUE INDEX `idx_host_infos_id` ON `host_infos` (`db_autopilot_id`, `db_host_id`); +CREATE INDEX `idx_host_infos_usability_blocked` ON `host_infos` (`usability_blocked`); +CREATE INDEX `idx_host_infos_usability_offline` ON `host_infos` (`usability_offline`); +CREATE INDEX `idx_host_infos_usability_low_score` ON `host_infos` (`usability_low_score`); +CREATE INDEX `idx_host_infos_usability_redundant_ip` ON `host_infos` (`usability_redundant_ip`); +CREATE INDEX `idx_host_infos_usability_gouging` ON `host_infos` (`usability_gouging`); +CREATE INDEX `idx_host_infos_usability_not_accepting_contracts` ON `host_infos` (`usability_not_accepting_contracts`); +CREATE INDEX `idx_host_infos_usability_not_announced` ON `host_infos` (`usability_not_announced`); +CREATE INDEX `idx_host_infos_usability_not_completing_scan` ON `host_infos` (`usability_not_completing_scan`); +CREATE INDEX `idx_host_infos_score_age` ON `host_infos` (`score_age`); +CREATE INDEX `idx_host_infos_score_collateral` ON `host_infos` (`score_collateral`); +CREATE INDEX `idx_host_infos_score_interactions` ON `host_infos` (`score_interactions`); +CREATE INDEX `idx_host_infos_score_storage_remaining` ON `host_infos` (`score_storage_remaining`); +CREATE INDEX `idx_host_infos_score_uptime` ON `host_infos` (`score_uptime`); +CREATE INDEX `idx_host_infos_score_version` ON `host_infos` (`score_version`); +CREATE INDEX `idx_host_infos_score_prices` ON `host_infos` (`score_prices`); + -- create default bucket INSERT INTO buckets (created_at, name) VALUES (CURRENT_TIMESTAMP, 'default'); From 3bf76384ec8cb76442c0a9f1d632d2b08d846f31 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 20 Mar 2024 14:19:24 +0100 Subject: [PATCH 02/16] stores: extend TestHostInfo with assertions for CASCADE DELETE --- stores/hostdb.go | 1 - stores/hostdb_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/stores/hostdb.go b/stores/hostdb.go index c344fd83f..5f7732fcd 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -648,7 +648,6 @@ func (ss *SQLStore) HostInfos(ctx context.Context, autopilotID string, filterMod // fetch host info var infos []dbHostInfo if err := query. - Debug(). Offset(offset). Limit(limit). Find(&infos). diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index 8a75caf6d..c9ab2ba7e 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -1247,6 +1247,37 @@ func TestHostInfo(t *testing.T) { } else if his[0].Host.PublicKey != (types.PublicKey{1}) { t.Fatal("unexpected", his) } + + // assert cascade delete on host + err = ss.db.Exec("DELETE FROM hosts WHERE public_key = ?", publicKey(types.PublicKey{1})).Error + if err != nil { + t.Fatal(err) + } + his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeUsable, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 0 { + t.Fatal("unexpected") + } + + // assert cascade delete on autopilot + var cnt uint64 + err = ss.db.Raw("SELECT COUNT(*) FROM host_infos").Scan(&cnt).Error + if err != nil { + t.Fatal(err) + } else if cnt == 0 { + t.Fatal("unexpected", cnt) + } + err = ss.db.Exec("DELETE FROM autopilots WHERE identifier = ?", "foo").Error + if err != nil { + t.Fatal(err) + } + err = ss.db.Raw("SELECT COUNT(*) FROM host_infos").Scan(&cnt).Error + if err != nil { + t.Fatal(err) + } else if cnt != 0 { + t.Fatal("unexpected", cnt) + } } // addTestHosts adds 'n' hosts to the db and returns their keys. From d74817ed2e8b248ba9598a304d2820b07ed218fd Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 21 Mar 2024 10:42:08 +0100 Subject: [PATCH 03/16] stores: add host scopes --- stores/hostdb.go | 312 +++++++++++++++++++++-------------------------- stores/sql.go | 24 +++- 2 files changed, 157 insertions(+), 179 deletions(-) diff --git a/stores/hostdb.go b/stores/hostdb.go index 5f7732fcd..5de891649 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -85,30 +85,30 @@ type ( dbHostInfo struct { Model - DBAutopilotID uint `gorm:"index:idx_host_infos_id,unique"` + DBAutopilotID uint DBAutopilot dbAutopilot - DBHostID uint `gorm:"index:idx_host_infos_id,unique"` + DBHostID uint DBHost dbHost // usability - UsabilityBlocked bool `gorm:"index:idx_host_infos_usability_blocked"` - UsabilityOffline bool `gorm:"index:idx_host_infos_usability_offline"` - UsabilityLowScore bool `gorm:"index:idx_host_infos_usability_low_score"` - UsabilityRedundantIP bool `gorm:"index:idx_host_infos_usability_redundant_ip"` - UsabilityGouging bool `gorm:"index:idx_host_infos_usability_gouging"` - UsabilityNotAcceptingContracts bool `gorm:"index:idx_host_infos_usability_not_accepting_contracts"` - UsabilityNotAnnounced bool `gorm:"index:idx_host_infos_usability_not_announced"` - UsabilityNotCompletingScan bool `gorm:"index:idx_host_infos_usability_not_completing_scan"` + UsabilityBlocked bool + UsabilityOffline bool + UsabilityLowScore bool + UsabilityRedundantIP bool + UsabilityGouging bool + UsabilityNotAcceptingContracts bool + UsabilityNotAnnounced bool + UsabilityNotCompletingScan bool // score - ScoreAge float64 `gorm:"index:idx_host_infos_score_age"` - ScoreCollateral float64 `gorm:"index:idx_host_infos_score_collateral"` - ScoreInteractions float64 `gorm:"index:idx_host_infos_score_interactions"` - ScoreStorageRemaining float64 `gorm:"index:idx_host_infos_score_storage_remaining"` - ScoreUptime float64 `gorm:"index:idx_host_infos_score_uptime"` - ScoreVersion float64 `gorm:"index:idx_host_infos_score_version"` - ScorePrices float64 `gorm:"index:idx_host_infos_score_prices"` + ScoreAge float64 + ScoreCollateral float64 + ScoreInteractions float64 + ScoreStorageRemaining float64 + ScoreUptime float64 + ScoreVersion float64 + ScorePrices float64 // gouging GougingContractErr string @@ -373,36 +373,6 @@ func (hi dbHostInfo) convert() api.HostInfo { } } -func convertHostInfo(apID, hID uint, gouging api.HostGougingBreakdown, score api.HostScoreBreakdown, usability api.HostUsabilityBreakdown) *dbHostInfo { - return &dbHostInfo{ - DBAutopilotID: apID, - DBHostID: hID, - - UsabilityBlocked: usability.Blocked, - UsabilityOffline: usability.Offline, - UsabilityLowScore: usability.LowScore, - UsabilityRedundantIP: usability.RedundantIP, - UsabilityGouging: usability.Gouging, - UsabilityNotAcceptingContracts: usability.NotAcceptingContracts, - UsabilityNotAnnounced: usability.NotAnnounced, - UsabilityNotCompletingScan: usability.NotCompletingScan, - - ScoreAge: score.Age, - ScoreCollateral: score.Collateral, - ScoreInteractions: score.Interactions, - ScoreStorageRemaining: score.StorageRemaining, - ScoreUptime: score.Uptime, - ScoreVersion: score.Version, - ScorePrices: score.Prices, - - GougingContractErr: gouging.ContractErr, - GougingDownloadErr: gouging.DownloadErr, - GougingGougingErr: gouging.GougingErr, - GougingPruneErr: gouging.PruneErr, - GougingUploadErr: gouging.UploadErr, - } -} - func (h *dbHost) BeforeCreate(tx *gorm.DB) (err error) { tx.Statement.AddClause(clause.OnConflict{ Columns: []clause.Column{{Name: "public_key"}}, @@ -531,45 +501,31 @@ func (ss *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (hostdb.H func (ss *SQLStore) HostInfo(ctx context.Context, autopilotID string, hk types.PublicKey) (hi api.HostInfo, err error) { err = ss.db.Transaction(func(tx *gorm.DB) error { - // fetch ap id - var apID uint - if err := tx. - Model(&dbAutopilot{}). - Where("identifier = ?", autopilotID). - Select("id"). - Take(&apID). - Error; errors.Is(err, gorm.ErrRecordNotFound) { - return api.ErrAutopilotNotFound - } else if err != nil { - return err - } - - // fetch host id - var hID uint - if err := tx. - Model(&dbHost{}). - Where("public_key = ?", publicKey(hk)). - Select("id"). - Take(&hID). - Error; errors.Is(err, gorm.ErrRecordNotFound) { - return api.ErrHostNotFound - } else if err != nil { - return err - } - - // fetch host info var entity dbHostInfo if err := tx. Model(&dbHostInfo{}). - Where("db_autopilot_id = ? AND db_host_id = ?", apID, hID). + Where("db_autopilot_id = (?)", gorm.Expr("SELECT id FROM autopilots WHERE identifier = ?", autopilotID)). + Where("db_host_id = (?)", gorm.Expr("SELECT id FROM hosts WHERE public_key = ?", publicKey(hk))). Preload("DBHost"). First(&entity). Error; errors.Is(err, gorm.ErrRecordNotFound) { + if err := tx. + Model(&dbAutopilot{}). + Where("identifier = ?", autopilotID). + First(nil). + Error; errors.Is(err, gorm.ErrRecordNotFound) { + return api.ErrAutopilotNotFound + } else if err := tx. + Model(&dbHost{}). + Where("public_key = ?", publicKey(hk)). + First(nil). + Error; errors.Is(err, gorm.ErrRecordNotFound) { + return api.ErrHostNotFound + } return api.ErrHostInfoNotFound } else if err != nil { return err } - hi = entity.convert() return nil }) @@ -601,49 +557,13 @@ func (ss *SQLStore) HostInfos(ctx context.Context, autopilotID string, filterMod Where("db_autopilot_id = ?", apID). Joins("DBHost") - // apply mode filter - switch filterMode { - case api.HostFilterModeAllowed: - query = query.Scopes(ss.excludeBlocked("DBHost")) - case api.HostFilterModeBlocked: - query = query.Scopes(ss.excludeAllowed("DBHost")) - case api.HostFilterModeAll: - // nothing to do - default: - return fmt.Errorf("invalid filter mode: %v", filterMode) - } - - // apply usability filter - switch usabilityMode { - case api.UsabilityFilterModeUsable: - query = query.Where("usability_blocked = ? AND usability_offline = ? AND usability_low_score = ? AND usability_redundant_ip = ? AND usability_gouging = ? AND usability_not_accepting_contracts = ? AND usability_not_announced = ? AND usability_not_completing_scan = ?", - false, false, false, false, false, false, false, false) - case api.UsabilityFilterModeUnusable: - query = query.Where("usability_blocked = ? OR usability_offline = ? OR usability_low_score = ? OR usability_redundant_ip = ? OR usability_gouging = ? OR usability_not_accepting_contracts = ? OR usability_not_announced = ? OR usability_not_completing_scan = ?", - true, true, true, true, true, true, true, true) - case api.UsabilityFilterModeAll: - // nothing to do - default: - return fmt.Errorf("invalid usability mode: %v", usabilityMode) - } - - // apply address filter - if addressContains != "" { - query = query.Scopes(func(d *gorm.DB) *gorm.DB { - return d.Where("net_address LIKE ?", "%"+addressContains+"%") - }) - } - - // apply key filter - if len(keyIn) > 0 { - pubKeys := make([]publicKey, len(keyIn)) - for i, pk := range keyIn { - pubKeys[i] = publicKey(pk) - } - query = query.Scopes(func(d *gorm.DB) *gorm.DB { - return d.Where("public_key IN ?", pubKeys) - }) - } + // apply filters + query = query.Scopes( + hostFilter(filterMode, ss.hasAllowlist(), ss.hasBlocklist(), "DBHost"), + hostUsabilityFilter(usabilityMode), + hostNetAddress(addressContains), + hostPublicKey(keyIn), + ) // fetch host info var infos []dbHostInfo @@ -696,7 +616,33 @@ func (ss *SQLStore) UpdateHostInfo(ctx context.Context, autopilotID string, hk t Columns: []clause.Column{{Name: "db_autopilot_id"}, {Name: "db_host_id"}}, UpdateAll: true, }). - Create(convertHostInfo(apID, hID, gouging, score, usability)). + Create(&dbHostInfo{ + DBAutopilotID: apID, + DBHostID: hID, + + UsabilityBlocked: usability.Blocked, + UsabilityOffline: usability.Offline, + UsabilityLowScore: usability.LowScore, + UsabilityRedundantIP: usability.RedundantIP, + UsabilityGouging: usability.Gouging, + UsabilityNotAcceptingContracts: usability.NotAcceptingContracts, + UsabilityNotAnnounced: usability.NotAnnounced, + UsabilityNotCompletingScan: usability.NotCompletingScan, + + ScoreAge: score.Age, + ScoreCollateral: score.Collateral, + ScoreInteractions: score.Interactions, + ScoreStorageRemaining: score.StorageRemaining, + ScoreUptime: score.Uptime, + ScoreVersion: score.Version, + ScorePrices: score.Prices, + + GougingContractErr: gouging.ContractErr, + GougingDownloadErr: gouging.DownloadErr, + GougingGougingErr: gouging.GougingErr, + GougingPruneErr: gouging.PruneErr, + GougingUploadErr: gouging.UploadErr, + }). Error }) return @@ -742,40 +688,27 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains return nil, ErrNegativeOffset } - // Apply filter mode. - var blocked bool - query := ss.db + // validate filterMode switch filterMode { case api.HostFilterModeAllowed: - query = query.Scopes(ss.excludeBlocked("hosts")) case api.HostFilterModeBlocked: - query = query.Scopes(ss.excludeAllowed("hosts")) - blocked = true case api.HostFilterModeAll: - // preload allowlist and blocklist - query = query. - Preload("Allowlist"). - Preload("Blocklist") default: return nil, fmt.Errorf("invalid filter mode: %v", filterMode) } - // Add address filter. - if addressContains != "" { - query = query.Scopes(func(d *gorm.DB) *gorm.DB { - return d.Where("net_address LIKE ?", "%"+addressContains+"%") - }) - } + // prepare query + query := ss.db.Scopes( + hostFilter(filterMode, ss.hasAllowlist(), ss.hasBlocklist(), "hosts"), + hostNetAddress(addressContains), + hostPublicKey(keyIn), + ) - // Only search for specific hosts. - if len(keyIn) > 0 { - pubKeys := make([]publicKey, len(keyIn)) - for i, pk := range keyIn { - pubKeys[i] = publicKey(pk) - } - query = query.Scopes(func(d *gorm.DB) *gorm.DB { - return d.Where("public_key IN ?", pubKeys) - }) + // preload allowlist and blocklist + if filterMode == api.HostFilterModeAll { + query = query. + Preload("Allowlist"). + Preload("Blocklist") } var hosts []hostdb.HostInfo @@ -793,7 +726,7 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains } else { hosts = append(hosts, hostdb.HostInfo{ Host: fh.convert(), - Blocked: blocked, + Blocked: filterMode == api.HostFilterModeBlocked, }) } } @@ -1206,40 +1139,73 @@ func (ss *SQLStore) processConsensusChangeHostDB(cc modules.ConsensusChange) { ss.unappliedAnnouncements = append(ss.unappliedAnnouncements, newAnnouncements...) } -// excludeBlocked can be used as a scope for a db transaction to exclude blocked -// hosts. -func (ss *SQLStore) excludeBlocked(alias string) func(db *gorm.DB) *gorm.DB { +// hostNetAddress can be used as a scope to filter hosts by their net address. +func hostNetAddress(addressContains string) func(*gorm.DB) *gorm.DB { return func(db *gorm.DB) *gorm.DB { - ss.mu.Lock() - defer ss.mu.Unlock() - - if ss.hasAllowlist { - db = db.Where(fmt.Sprintf("EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", alias)) - } - if ss.hasBlocklist { - db = db.Where(fmt.Sprintf("NOT EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", alias)) + if addressContains != "" { + return db.Where("net_address LIKE ?", "%"+addressContains+"%") } return db } } -// excludeAllowed can be used as a scope for a db transaction to exclude allowed -// hosts. -func (ss *SQLStore) excludeAllowed(alias string) func(db *gorm.DB) *gorm.DB { +func hostPublicKey(keyIn []types.PublicKey) func(*gorm.DB) *gorm.DB { return func(db *gorm.DB) *gorm.DB { - ss.mu.Lock() - defer ss.mu.Unlock() - - if ss.hasAllowlist { - db = db.Where(fmt.Sprintf("NOT EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", alias)) + if len(keyIn) > 0 { + pubKeys := make([]publicKey, len(keyIn)) + for i, pk := range keyIn { + pubKeys[i] = publicKey(pk) + } + return db.Where("public_key IN ?", pubKeys) } - if ss.hasBlocklist { - db = db.Where(fmt.Sprintf("EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", alias)) + return db + } +} + +// hostFilter can be used as a scope to filter hosts based on their filter mode, +// returning either all, allowed or blocked hosts. +func hostFilter(filterMode string, hasAllowlist, hasBlocklist bool, hostTableAlias string) func(*gorm.DB) *gorm.DB { + return func(db *gorm.DB) *gorm.DB { + switch filterMode { + case api.HostFilterModeAllowed: + if hasAllowlist { + db = db.Where(fmt.Sprintf("EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", hostTableAlias)) + } + if hasBlocklist { + db = db.Where(fmt.Sprintf("NOT EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", hostTableAlias)) + } + case api.HostFilterModeBlocked: + if hasAllowlist { + db = db.Where(fmt.Sprintf("NOT EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", hostTableAlias)) + } + if hasBlocklist { + db = db.Where(fmt.Sprintf("EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", hostTableAlias)) + } + if !hasAllowlist && !hasBlocklist { + // if neither an allowlist nor a blocklist exist, all hosts are allowed + // which means we return none + db = db.Where("1 = 0") + } + case api.HostFilterModeAll: + // do nothing } - if !ss.hasAllowlist && !ss.hasBlocklist { - // if neither an allowlist nor a blocklist exist, all hosts are allowed - // which means we return none - db = db.Where("1 = 0") + return db + } +} + +// hostUsabilityFilter can be used as a scope to filter hosts based on their +// usability mode, return either all, usable or unusable hosts. hosts. +func hostUsabilityFilter(usabilityMode string) func(*gorm.DB) *gorm.DB { + return func(db *gorm.DB) *gorm.DB { + switch usabilityMode { + case api.UsabilityFilterModeUsable: + db = db.Where("usability_blocked = ? AND usability_offline = ? AND usability_low_score = ? AND usability_redundant_ip = ? AND usability_gouging = ? AND usability_not_accepting_contracts = ? AND usability_not_announced = ? AND usability_not_completing_scan = ?", + false, false, false, false, false, false, false, false) + case api.UsabilityFilterModeUnusable: + db.Where("usability_blocked = ? OR usability_offline = ? OR usability_low_score = ? OR usability_redundant_ip = ? OR usability_gouging = ? OR usability_not_accepting_contracts = ? OR usability_not_announced = ? OR usability_not_completing_scan = ?", + true, true, true, true, true, true, true, true) + case api.UsabilityFilterModeAll: + // nothing to do } return db } @@ -1249,10 +1215,10 @@ func (ss *SQLStore) isBlocked(h dbHost) (blocked bool) { ss.mu.Lock() defer ss.mu.Unlock() - if ss.hasAllowlist && len(h.Allowlist) == 0 { + if ss.allowListCnt > 0 && len(h.Allowlist) == 0 { blocked = true } - if ss.hasBlocklist && len(h.Blocklist) > 0 { + if ss.blockListCnt > 0 && len(h.Blocklist) > 0 { blocked = true } return diff --git a/stores/sql.go b/stores/sql.go index f62dba97f..34a6d78ab 100644 --- a/stores/sql.go +++ b/stores/sql.go @@ -104,8 +104,8 @@ type ( wg sync.WaitGroup mu sync.Mutex - hasAllowlist bool - hasBlocklist bool + allowListCnt uint64 + blockListCnt uint64 closed bool knownContracts map[types.FileContractID]struct{} @@ -258,8 +258,8 @@ func NewSQLStore(cfg Config) (*SQLStore, modules.ConsensusChangeID, error) { knownContracts: isOurContract, lastSave: time.Now(), persistInterval: cfg.PersistInterval, - hasAllowlist: allowlistCnt > 0, - hasBlocklist: blocklistCnt > 0, + allowListCnt: uint64(allowlistCnt), + blockListCnt: uint64(blocklistCnt), settings: make(map[string]string), slabPruneSigChan: make(chan struct{}, 1), unappliedContractState: make(map[types.FileContractID]contractState), @@ -299,6 +299,12 @@ func isSQLite(db *gorm.DB) bool { } } +func (ss *SQLStore) hasAllowlist() bool { + ss.mu.Lock() + defer ss.mu.Unlock() + return ss.allowListCnt > 0 +} + func (ss *SQLStore) updateHasAllowlist(err *error) { if *err != nil { return @@ -311,10 +317,16 @@ func (ss *SQLStore) updateHasAllowlist(err *error) { } ss.mu.Lock() - ss.hasAllowlist = cnt > 0 + ss.allowListCnt = uint64(cnt) ss.mu.Unlock() } +func (ss *SQLStore) hasBlocklist() bool { + ss.mu.Lock() + defer ss.mu.Unlock() + return ss.blockListCnt > 0 +} + func (ss *SQLStore) updateHasBlocklist(err *error) { if *err != nil { return @@ -327,7 +339,7 @@ func (ss *SQLStore) updateHasBlocklist(err *error) { } ss.mu.Lock() - ss.hasBlocklist = cnt > 0 + ss.blockListCnt = uint64(cnt) ss.mu.Unlock() } From df6641510e7deef0045b6f512d0058080da2a8be Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 21 Mar 2024 20:33:22 +0100 Subject: [PATCH 04/16] stores: unify HostInfos and SearchHosts --- api/host.go | 13 +- autopilot/autopilot.go | 16 +- autopilot/autopilot_test.go | 50 +-- autopilot/client.go | 18 +- autopilot/contractor.go | 12 +- autopilot/hostinfo.go | 7 +- autopilot/scanner.go | 2 +- autopilot/scanner_test.go | 10 +- bus/bus.go | 10 +- bus/client/hosts.go | 4 +- stores/hostdb.go | 231 ++++------- stores/hostdb_test.go | 375 +++++++----------- stores/metadata_test.go | 4 +- stores/migrations.go | 4 +- .../main/migration_00007_host_checks.sql | 52 +++ .../mysql/main/migration_00007_host_info.sql | 52 --- stores/migrations/mysql/main/schema.sql | 42 +- .../main/migration_00007_host_checks.sql | 52 +++ .../sqlite/main/migration_00007_host_info.sql | 52 --- stores/migrations/sqlite/main/schema.sql | 36 +- worker/mocks_test.go | 15 +- worker/worker.go | 2 +- 22 files changed, 468 insertions(+), 591 deletions(-) create mode 100644 stores/migrations/mysql/main/migration_00007_host_checks.sql delete mode 100644 stores/migrations/mysql/main/migration_00007_host_info.sql create mode 100644 stores/migrations/sqlite/main/migration_00007_host_checks.sql delete mode 100644 stores/migrations/sqlite/main/migration_00007_host_info.sql diff --git a/api/host.go b/api/host.go index 2c1e82f30..50b058642 100644 --- a/api/host.go +++ b/api/host.go @@ -47,11 +47,15 @@ type ( MinRecentScanFailures uint64 `json:"minRecentScanFailures"` } + HostsRequest struct { + UsabilityMode string `json:"usabilityMode"` + SearchHostsRequest + } + SearchHostsRequest struct { Offset int `json:"offset"` Limit int `json:"limit"` FilterMode string `json:"filterMode"` - UsabilityMode string `json:"usabilityMode"` AddressContains string `json:"addressContains"` KeyIn []types.PublicKey `json:"keyIn"` } @@ -88,6 +92,7 @@ type ( SearchHostOptions struct { AddressContains string FilterMode string + UsabilityMode string KeyIn []types.PublicKey Limit int Offset int @@ -117,7 +122,11 @@ func (opts HostsForScanningOptions) Apply(values url.Values) { type ( HostInfo struct { - Host hostdb.Host `json:"host"` + hostdb.HostInfo + Checks map[string]HostCheck `json:"checks"` + } + + HostCheck struct { Gouging HostGougingBreakdown `json:"gouging"` Score HostScoreBreakdown `json:"score"` Usability HostUsabilityBreakdown `json:"usability"` diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 1038f4378..7562de960 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -53,10 +53,10 @@ type Bus interface { PrunableData(ctx context.Context) (prunableData api.ContractsPrunableDataResponse, err error) // hostdb - Host(ctx context.Context, hostKey types.PublicKey) (hostdb.HostInfo, error) + Host(ctx context.Context, hostKey types.PublicKey) (api.HostInfo, error) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error) RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) - SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.HostInfo, error) + SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.HostInfo, error) // metrics RecordContractSetChurnMetric(ctx context.Context, metrics ...api.ContractSetChurnMetric) error @@ -726,7 +726,7 @@ func (ap *Autopilot) stateHandlerGET(jc jape.Context) { } func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { - var req api.SearchHostsRequest + var req api.HostsRequest if jc.Decode(&req) != nil { return } @@ -737,10 +737,10 @@ func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { jc.Encode(hosts) } -func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.HostInfo) (usables uint64) { +func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.HostInfo) (usables uint64) { gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow) for _, host := range hosts { - usable, _ := isUsableHost(cfg, rs, gc, host, smallestValidScore, 0) + usable, _ := isUsableHost(cfg, rs, gc, host.HostInfo, smallestValidScore, 0) if usable { usables++ } @@ -751,12 +751,12 @@ func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types. // evaluateConfig evaluates the given configuration and if the gouging settings // are too strict for the number of contracts required by 'cfg', it will provide // a recommendation on how to loosen it. -func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.HostInfo) (resp api.ConfigEvaluationResponse) { +func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.HostInfo) (resp api.ConfigEvaluationResponse) { gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow) resp.Hosts = uint64(len(hosts)) for _, host := range hosts { - usable, usableBreakdown := isUsableHost(cfg, rs, gc, host, 0, 0) + usable, usableBreakdown := isUsableHost(cfg, rs, gc, host.HostInfo, 0, 0) if usable { resp.Usable++ continue @@ -866,7 +866,7 @@ func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Cu // optimiseGougingSetting tries to optimise one field of the gouging settings to // try and hit the target number of contracts. -func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, hosts []hostdb.HostInfo) bool { +func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, hosts []api.HostInfo) bool { if cfg.Contracts.Amount == 0 { return true // nothing to do } diff --git a/autopilot/autopilot_test.go b/autopilot/autopilot_test.go index 9ebafe675..2da3fc7be 100644 --- a/autopilot/autopilot_test.go +++ b/autopilot/autopilot_test.go @@ -14,34 +14,36 @@ import ( func TestOptimiseGougingSetting(t *testing.T) { // create 10 hosts that should all be usable - var hosts []hostdb.HostInfo + var hosts []api.HostInfo for i := 0; i < 10; i++ { - hosts = append(hosts, hostdb.HostInfo{ - Host: hostdb.Host{ - KnownSince: time.Unix(0, 0), - PriceTable: hostdb.HostPriceTable{ - HostPriceTable: rhpv3.HostPriceTable{ - CollateralCost: types.Siacoins(1), - MaxCollateral: types.Siacoins(1000), + hosts = append(hosts, api.HostInfo{ + HostInfo: hostdb.HostInfo{ + Host: hostdb.Host{ + KnownSince: time.Unix(0, 0), + PriceTable: hostdb.HostPriceTable{ + HostPriceTable: rhpv3.HostPriceTable{ + CollateralCost: types.Siacoins(1), + MaxCollateral: types.Siacoins(1000), + }, }, + Settings: rhpv2.HostSettings{ + AcceptingContracts: true, + Collateral: types.Siacoins(1), + MaxCollateral: types.Siacoins(1000), + Version: "1.6.0", + }, + Interactions: hostdb.Interactions{ + Uptime: time.Hour * 1000, + LastScan: time.Now(), + LastScanSuccess: true, + SecondToLastScanSuccess: true, + TotalScans: 100, + }, + LastAnnouncement: time.Unix(0, 0), + Scanned: true, }, - Settings: rhpv2.HostSettings{ - AcceptingContracts: true, - Collateral: types.Siacoins(1), - MaxCollateral: types.Siacoins(1000), - Version: "1.6.0", - }, - Interactions: hostdb.Interactions{ - Uptime: time.Hour * 1000, - LastScan: time.Now(), - LastScanSuccess: true, - SecondToLastScanSuccess: true, - TotalScans: 100, - }, - LastAnnouncement: time.Unix(0, 0), - Scanned: true, + Blocked: false, }, - Blocked: false, }) } diff --git a/autopilot/client.go b/autopilot/client.go index ba16754a5..336149f8a 100644 --- a/autopilot/client.go +++ b/autopilot/client.go @@ -40,14 +40,16 @@ func (c *Client) HostInfo(hostKey types.PublicKey) (resp api.HostHandlerResponse } // HostInfo returns information about all hosts. -func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode string, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostHandlerResponse, err error) { - err = c.c.POST("/hosts", api.SearchHostsRequest{ - Offset: offset, - Limit: limit, - FilterMode: filterMode, - UsabilityMode: usabilityMode, - AddressContains: addressContains, - KeyIn: keyIn, +func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostHandlerResponse, err error) { + err = c.c.POST("/hosts", api.HostsRequest{ + UsabilityMode: usabilityMode, + SearchHostsRequest: api.SearchHostsRequest{ + Offset: offset, + Limit: limit, + FilterMode: filterMode, + AddressContains: addressContains, + KeyIn: keyIn, + }, }, &resp) return } diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 43ac5e629..d376d682e 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -295,7 +295,7 @@ func (c *contractor) performContractMaintenance(ctx context.Context, w Worker) ( for _, h := range hosts { // ignore the pricetable's HostBlockHeight by setting it to our own blockheight h.PriceTable.HostBlockHeight = cs.BlockHeight - isUsable, unusableResult := isUsableHost(state.cfg, state.rs, gc, h, minScore, hostData[h.PublicKey]) + isUsable, unusableResult := isUsableHost(state.cfg, state.rs, gc, h.HostInfo, minScore, hostData[h.PublicKey]) hostInfos[h.PublicKey] = hostInfo{ Usable: isUsable, UnusableResult: unusableResult, @@ -777,7 +777,7 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts host.PriceTable.HostBlockHeight = cs.BlockHeight // decide whether the host is still good - usable, unusableResult := isUsableHost(state.cfg, state.rs, gc, host, minScore, contract.FileSize()) + usable, unusableResult := isUsableHost(state.cfg, state.rs, gc, host.HostInfo, minScore, contract.FileSize()) if !usable { reasons := unusableResult.reasons() toStopUsing[fcid] = strings.Join(reasons, ",") @@ -1297,7 +1297,7 @@ func (c *contractor) calculateMinScore(candidates []scoredHost, numContracts uin return minScore } -func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.HostInfo, usedHosts map[types.PublicKey]struct{}, storedData map[types.PublicKey]uint64, minScore float64) ([]scoredHost, unusableHostResult, error) { +func (c *contractor) candidateHosts(ctx context.Context, hosts []api.HostInfo, usedHosts map[types.PublicKey]struct{}, storedData map[types.PublicKey]uint64, minScore float64) ([]scoredHost, unusableHostResult, error) { start := time.Now() // fetch consensus state @@ -1311,7 +1311,7 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.HostInfo gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) // select unused hosts that passed a scan - var unused []hostdb.HostInfo + var unused []api.HostInfo var excluded, notcompletedscan int for _, h := range hosts { // filter out used hosts @@ -1346,7 +1346,7 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.HostInfo // NOTE: ignore the pricetable's HostBlockHeight by setting it to our // own blockheight h.PriceTable.HostBlockHeight = cs.BlockHeight - usable, result := isUsableHost(state.cfg, state.rs, gc, h, minScore, storedData[h.PublicKey]) + usable, result := isUsableHost(state.cfg, state.rs, gc, h.HostInfo, minScore, storedData[h.PublicKey]) if usable { candidates = append(candidates, scoredHost{h.Host, result.scoreBreakdown.Score()}) continue @@ -1612,7 +1612,7 @@ func (c *contractor) tryPerformPruning(wp *workerPool) { }() } -func (c *contractor) hostForContract(ctx context.Context, fcid types.FileContractID) (host hostdb.HostInfo, metadata api.ContractMetadata, err error) { +func (c *contractor) hostForContract(ctx context.Context, fcid types.FileContractID) (host api.HostInfo, metadata api.ContractMetadata, err error) { // fetch the contract metadata, err = c.ap.bus.Contract(ctx, fcid) if err != nil { diff --git a/autopilot/hostinfo.go b/autopilot/hostinfo.go index e0cbecadc..5af554e6b 100644 --- a/autopilot/hostinfo.go +++ b/autopilot/hostinfo.go @@ -6,7 +6,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/worker" ) @@ -53,7 +52,7 @@ func (c *contractor) HostInfo(ctx context.Context, hostKey types.PublicKey) (api // ignore the pricetable's HostBlockHeight by setting it to our own blockheight host.Host.PriceTable.HostBlockHeight = cs.BlockHeight - isUsable, unusableResult := isUsableHost(state.cfg, rs, gc, host, minScore, storedData) + isUsable, unusableResult := isUsableHost(state.cfg, rs, gc, host.HostInfo, minScore, storedData) return api.HostHandlerResponse{ Host: host.Host, Checks: &api.HostHandlerResponseChecks{ @@ -67,7 +66,7 @@ func (c *contractor) HostInfo(ctx context.Context, hostKey types.PublicKey) (api }, nil } -func (c *contractor) hostInfoFromCache(ctx context.Context, host hostdb.HostInfo) (hi hostInfo, found bool) { +func (c *contractor) hostInfoFromCache(ctx context.Context, host api.HostInfo) (hi hostInfo, found bool) { // grab host details from cache c.mu.Lock() hi, found = c.cachedHostInfo[host.PublicKey] @@ -90,7 +89,7 @@ func (c *contractor) hostInfoFromCache(ctx context.Context, host hostdb.HostInfo } else { state := c.ap.State() gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) - isUsable, unusableResult := isUsableHost(state.cfg, state.rs, gc, host, minScore, storedData) + isUsable, unusableResult := isUsableHost(state.cfg, state.rs, gc, host.HostInfo, minScore, storedData) hi = hostInfo{ Usable: isUsable, UnusableResult: unusableResult, diff --git a/autopilot/scanner.go b/autopilot/scanner.go index 76643e5b5..a2d30abfa 100644 --- a/autopilot/scanner.go +++ b/autopilot/scanner.go @@ -31,7 +31,7 @@ type ( // a bit, we currently use inline interfaces to avoid having to update the // scanner tests with every interface change bus interface { - SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.HostInfo, error) + SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.HostInfo, error) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error) RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) } diff --git a/autopilot/scanner_test.go b/autopilot/scanner_test.go index 1cdd096d2..027366662 100644 --- a/autopilot/scanner_test.go +++ b/autopilot/scanner_test.go @@ -19,7 +19,7 @@ type mockBus struct { reqs []string } -func (b *mockBus) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.HostInfo, error) { +func (b *mockBus) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.HostInfo, error) { b.reqs = append(b.reqs, fmt.Sprintf("%d-%d", opts.Offset, opts.Offset+opts.Limit)) start := opts.Offset @@ -32,9 +32,13 @@ func (b *mockBus) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ( end = len(b.hosts) } - his := make([]hostdb.HostInfo, len(b.hosts[start:end])) + his := make([]api.HostInfo, len(b.hosts[start:end])) for i, h := range b.hosts[start:end] { - his[i] = hostdb.HostInfo{Host: h} + his[i] = api.HostInfo{ + HostInfo: hostdb.HostInfo{ + Host: h, + }, + } } return his, nil } diff --git a/bus/bus.go b/bus/bus.go index 7d33964be..510353edd 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -91,13 +91,13 @@ type ( // A HostDB stores information about hosts. HostDB interface { - Host(ctx context.Context, hostKey types.PublicKey) (hostdb.HostInfo, error) + Host(ctx context.Context, hostKey types.PublicKey) (api.HostInfo, error) HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]hostdb.HostAddress, error) RecordHostScans(ctx context.Context, scans []hostdb.HostScan) error RecordPriceTables(ctx context.Context, priceTableUpdate []hostdb.PriceTableUpdate) error RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) ResetLostSectors(ctx context.Context, hk types.PublicKey) error - SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]hostdb.HostInfo, error) + SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.HostInfo, error) HostAllowlist(ctx context.Context) ([]types.PublicKey, error) HostBlocklist(ctx context.Context) ([]string, error) @@ -775,9 +775,9 @@ func (b *bus) searchHostsHandlerPOST(jc jape.Context) { return } - // TODO: on the next major release - // - set defaults in handler - // - validate request params and return 400 if invalid + // TODO: on the next major release: + // - properly default search params + // - properly validate and return 400 hosts, err := b.hdb.SearchHosts(jc.Request.Context(), req.FilterMode, req.AddressContains, req.KeyIn, req.Offset, req.Limit) if jc.Check(fmt.Sprintf("couldn't fetch hosts %d-%d", req.Offset, req.Offset+req.Limit), err) != nil { return diff --git a/bus/client/hosts.go b/bus/client/hosts.go index 1ebf14e1f..460291dd6 100644 --- a/bus/client/hosts.go +++ b/bus/client/hosts.go @@ -12,7 +12,7 @@ import ( ) // Host returns information about a particular host known to the server. -func (c *Client) Host(ctx context.Context, hostKey types.PublicKey) (h hostdb.HostInfo, err error) { +func (c *Client) Host(ctx context.Context, hostKey types.PublicKey) (h api.HostInfo, err error) { err = c.c.WithContext(ctx).GET(fmt.Sprintf("/host/%s", hostKey), &h) return } @@ -78,7 +78,7 @@ func (c *Client) ResetLostSectors(ctx context.Context, hostKey types.PublicKey) } // SearchHosts returns all hosts that match certain search criteria. -func (c *Client) SearchHosts(ctx context.Context, opts api.SearchHostOptions) (hosts []hostdb.HostInfo, err error) { +func (c *Client) SearchHosts(ctx context.Context, opts api.SearchHostOptions) (hosts []api.HostInfo, err error) { err = c.c.WithContext(ctx).POST("/search/hosts", api.SearchHostsRequest{ Offset: opts.Offset, Limit: opts.Limit, diff --git a/stores/hostdb.go b/stores/hostdb.go index 5de891649..01a9e594e 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -78,11 +78,12 @@ type ( Allowlist []dbAllowlistEntry `gorm:"many2many:host_allowlist_entry_hosts;constraint:OnDelete:CASCADE"` Blocklist []dbBlocklistEntry `gorm:"many2many:host_blocklist_entry_hosts;constraint:OnDelete:CASCADE"` + Checks []dbHostCheck `gorm:"foreignKey:DBHostID;constraint:OnDelete:CASCADE"` } - // dbHostInfo contains information about a host that is collected and used + // dbHostCheck contains information about a host that is collected and used // by the autopilot. - dbHostInfo struct { + dbHostCheck struct { Model DBAutopilotID uint @@ -302,7 +303,7 @@ func (dbConsensusInfo) TableName() string { return "consensus_infos" } func (dbHost) TableName() string { return "hosts" } // TableName implements the gorm.Tabler interface. -func (dbHostInfo) TableName() string { return "host_infos" } +func (dbHostCheck) TableName() string { return "host_checks" } // TableName implements the gorm.Tabler interface. func (dbAllowlistEntry) TableName() string { return "host_allowlist_entries" } @@ -310,40 +311,49 @@ func (dbAllowlistEntry) TableName() string { return "host_allowlist_entries" } // TableName implements the gorm.Tabler interface. func (dbBlocklistEntry) TableName() string { return "host_blocklist_entries" } -// convert converts a host into a hostdb.Host. -func (h dbHost) convert() hostdb.Host { +// convert converts a host into a api.HostInfo +func (h dbHost) convert(blocked bool) api.HostInfo { var lastScan time.Time if h.LastScan > 0 { lastScan = time.Unix(0, h.LastScan) } - return hostdb.Host{ - KnownSince: h.CreatedAt, - LastAnnouncement: h.LastAnnouncement, - NetAddress: h.NetAddress, - Interactions: hostdb.Interactions{ - TotalScans: h.TotalScans, - LastScan: lastScan, - LastScanSuccess: h.LastScanSuccess, - SecondToLastScanSuccess: h.SecondToLastScanSuccess, - Uptime: h.Uptime, - Downtime: h.Downtime, - SuccessfulInteractions: h.SuccessfulInteractions, - FailedInteractions: h.FailedInteractions, - LostSectors: h.LostSectors, - }, - PriceTable: hostdb.HostPriceTable{ - HostPriceTable: h.PriceTable.convert(), - Expiry: h.PriceTableExpiry.Time, + checks := make(map[string]api.HostCheck) + for _, check := range h.Checks { + checks[check.DBAutopilot.Identifier] = check.convert() + } + return api.HostInfo{ + HostInfo: hostdb.HostInfo{ + Host: hostdb.Host{ + KnownSince: h.CreatedAt, + LastAnnouncement: h.LastAnnouncement, + NetAddress: h.NetAddress, + Interactions: hostdb.Interactions{ + TotalScans: h.TotalScans, + LastScan: lastScan, + LastScanSuccess: h.LastScanSuccess, + SecondToLastScanSuccess: h.SecondToLastScanSuccess, + Uptime: h.Uptime, + Downtime: h.Downtime, + SuccessfulInteractions: h.SuccessfulInteractions, + FailedInteractions: h.FailedInteractions, + LostSectors: h.LostSectors, + }, + PriceTable: hostdb.HostPriceTable{ + HostPriceTable: h.PriceTable.convert(), + Expiry: h.PriceTableExpiry.Time, + }, + PublicKey: types.PublicKey(h.PublicKey), + Scanned: h.Scanned, + Settings: h.Settings.convert(), + }, + Blocked: blocked, }, - PublicKey: types.PublicKey(h.PublicKey), - Scanned: h.Scanned, - Settings: h.Settings.convert(), + Checks: checks, } } -func (hi dbHostInfo) convert() api.HostInfo { - return api.HostInfo{ - Host: hi.DBHost.convert(), +func (hi dbHostCheck) convert() api.HostCheck { + return api.HostCheck{ Gouging: api.HostGougingBreakdown{ ContractErr: hi.GougingContractErr, DownloadErr: hi.GougingDownloadErr, @@ -478,7 +488,7 @@ func (e *dbBlocklistEntry) blocks(h dbHost) bool { } // Host returns information about a host. -func (ss *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (hostdb.HostInfo, error) { +func (ss *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (api.HostInfo, error) { var h dbHost tx := ss.db. @@ -488,22 +498,19 @@ func (ss *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (hostdb.H Preload("Blocklist"). Take(&h) if errors.Is(tx.Error, gorm.ErrRecordNotFound) { - return hostdb.HostInfo{}, api.ErrHostNotFound + return api.HostInfo{}, api.ErrHostNotFound } else if tx.Error != nil { - return hostdb.HostInfo{}, tx.Error + return api.HostInfo{}, tx.Error } - return hostdb.HostInfo{ - Host: h.convert(), - Blocked: ss.isBlocked(h), - }, nil + return h.convert(ss.isBlocked(h)), nil } func (ss *SQLStore) HostInfo(ctx context.Context, autopilotID string, hk types.PublicKey) (hi api.HostInfo, err error) { err = ss.db.Transaction(func(tx *gorm.DB) error { - var entity dbHostInfo + var entity dbHostCheck if err := tx. - Model(&dbHostInfo{}). + Model(&dbHostCheck{}). Where("db_autopilot_id = (?)", gorm.Expr("SELECT id FROM autopilots WHERE identifier = ?", autopilotID)). Where("db_host_id = (?)", gorm.Expr("SELECT id FROM hosts WHERE public_key = ?", publicKey(hk))). Preload("DBHost"). @@ -526,63 +533,13 @@ func (ss *SQLStore) HostInfo(ctx context.Context, autopilotID string, hk types.P } else if err != nil { return err } - hi = entity.convert() - return nil - }) - return -} - -func (ss *SQLStore) HostInfos(ctx context.Context, autopilotID string, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) (his []api.HostInfo, err error) { - if offset < 0 { - return nil, ErrNegativeOffset - } - - err = ss.db.Transaction(func(tx *gorm.DB) error { - // fetch ap id - var apID uint - if err := tx. - Model(&dbAutopilot{}). - Where("identifier = ?", autopilotID). - Select("id"). - Take(&apID). - Error; errors.Is(err, gorm.ErrRecordNotFound) { - return api.ErrAutopilotNotFound - } else if err != nil { - return err - } - - // prepare query - query := tx. - Model(&dbHostInfo{}). - Where("db_autopilot_id = ?", apID). - Joins("DBHost") - - // apply filters - query = query.Scopes( - hostFilter(filterMode, ss.hasAllowlist(), ss.hasBlocklist(), "DBHost"), - hostUsabilityFilter(usabilityMode), - hostNetAddress(addressContains), - hostPublicKey(keyIn), - ) - - // fetch host info - var infos []dbHostInfo - if err := query. - Offset(offset). - Limit(limit). - Find(&infos). - Error; err != nil { - return err - } - for _, hi := range infos { - his = append(his, hi.convert()) - } + // hi = entity.convert() return nil }) return } -func (ss *SQLStore) UpdateHostInfo(ctx context.Context, autopilotID string, hk types.PublicKey, gouging api.HostGougingBreakdown, score api.HostScoreBreakdown, usability api.HostUsabilityBreakdown) (err error) { +func (ss *SQLStore) UpdateHostCheck(ctx context.Context, autopilotID string, hk types.PublicKey, hc api.HostCheck) (err error) { err = ss.db.Transaction(func(tx *gorm.DB) error { // fetch ap id var apID uint @@ -616,32 +573,32 @@ func (ss *SQLStore) UpdateHostInfo(ctx context.Context, autopilotID string, hk t Columns: []clause.Column{{Name: "db_autopilot_id"}, {Name: "db_host_id"}}, UpdateAll: true, }). - Create(&dbHostInfo{ + Create(&dbHostCheck{ DBAutopilotID: apID, DBHostID: hID, - UsabilityBlocked: usability.Blocked, - UsabilityOffline: usability.Offline, - UsabilityLowScore: usability.LowScore, - UsabilityRedundantIP: usability.RedundantIP, - UsabilityGouging: usability.Gouging, - UsabilityNotAcceptingContracts: usability.NotAcceptingContracts, - UsabilityNotAnnounced: usability.NotAnnounced, - UsabilityNotCompletingScan: usability.NotCompletingScan, - - ScoreAge: score.Age, - ScoreCollateral: score.Collateral, - ScoreInteractions: score.Interactions, - ScoreStorageRemaining: score.StorageRemaining, - ScoreUptime: score.Uptime, - ScoreVersion: score.Version, - ScorePrices: score.Prices, - - GougingContractErr: gouging.ContractErr, - GougingDownloadErr: gouging.DownloadErr, - GougingGougingErr: gouging.GougingErr, - GougingPruneErr: gouging.PruneErr, - GougingUploadErr: gouging.UploadErr, + UsabilityBlocked: hc.Usability.Blocked, + UsabilityOffline: hc.Usability.Offline, + UsabilityLowScore: hc.Usability.LowScore, + UsabilityRedundantIP: hc.Usability.RedundantIP, + UsabilityGouging: hc.Usability.Gouging, + UsabilityNotAcceptingContracts: hc.Usability.NotAcceptingContracts, + UsabilityNotAnnounced: hc.Usability.NotAnnounced, + UsabilityNotCompletingScan: hc.Usability.NotCompletingScan, + + ScoreAge: hc.Score.Age, + ScoreCollateral: hc.Score.Collateral, + ScoreInteractions: hc.Score.Interactions, + ScoreStorageRemaining: hc.Score.StorageRemaining, + ScoreUptime: hc.Score.Uptime, + ScoreVersion: hc.Score.Version, + ScorePrices: hc.Score.Prices, + + GougingContractErr: hc.Gouging.ContractErr, + GougingDownloadErr: hc.Gouging.DownloadErr, + GougingGougingErr: hc.Gouging.GougingErr, + GougingPruneErr: hc.Gouging.PruneErr, + GougingUploadErr: hc.Gouging.UploadErr, }). Error }) @@ -683,7 +640,7 @@ func (ss *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time, return hostAddresses, err } -func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]hostdb.HostInfo, error) { +func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.HostInfo, error) { if offset < 0 { return nil, ErrNegativeOffset } @@ -698,11 +655,13 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains } // prepare query - query := ss.db.Scopes( - hostFilter(filterMode, ss.hasAllowlist(), ss.hasBlocklist(), "hosts"), - hostNetAddress(addressContains), - hostPublicKey(keyIn), - ) + query := ss.db. + Model(&dbHost{}). + Scopes( + hostFilter(filterMode, ss.hasAllowlist(), ss.hasBlocklist(), "hosts"), + hostNetAddress(addressContains), + hostPublicKey(keyIn), + ).Preload("Checks.DBAutopilot") // preload allowlist and blocklist if filterMode == api.HostFilterModeAll { @@ -711,24 +670,20 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains Preload("Blocklist") } - var hosts []hostdb.HostInfo + var hosts []api.HostInfo var fullHosts []dbHost err := query. Offset(offset). Limit(limit). FindInBatches(&fullHosts, hostRetrievalBatchSize, func(tx *gorm.DB, batch int) error { for _, fh := range fullHosts { + var blocked bool if filterMode == api.HostFilterModeAll { - hosts = append(hosts, hostdb.HostInfo{ - Host: fh.convert(), - Blocked: ss.isBlocked(fh), - }) + blocked = ss.isBlocked(fh) } else { - hosts = append(hosts, hostdb.HostInfo{ - Host: fh.convert(), - Blocked: filterMode == api.HostFilterModeBlocked, - }) + blocked = filterMode == api.HostFilterModeBlocked } + hosts = append(hosts, fh.convert(blocked)) } return nil }). @@ -740,7 +695,7 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains } // Hosts returns non-blocked hosts at given offset and limit. -func (ss *SQLStore) Hosts(ctx context.Context, offset, limit int) ([]hostdb.HostInfo, error) { +func (ss *SQLStore) Hosts(ctx context.Context, offset, limit int) ([]api.HostInfo, error) { return ss.SearchHosts(ctx, api.HostFilterModeAllowed, "", nil, offset, limit) } @@ -1193,24 +1148,6 @@ func hostFilter(filterMode string, hasAllowlist, hasBlocklist bool, hostTableAli } } -// hostUsabilityFilter can be used as a scope to filter hosts based on their -// usability mode, return either all, usable or unusable hosts. hosts. -func hostUsabilityFilter(usabilityMode string) func(*gorm.DB) *gorm.DB { - return func(db *gorm.DB) *gorm.DB { - switch usabilityMode { - case api.UsabilityFilterModeUsable: - db = db.Where("usability_blocked = ? AND usability_offline = ? AND usability_low_score = ? AND usability_redundant_ip = ? AND usability_gouging = ? AND usability_not_accepting_contracts = ? AND usability_not_announced = ? AND usability_not_completing_scan = ?", - false, false, false, false, false, false, false, false) - case api.UsabilityFilterModeUnusable: - db.Where("usability_blocked = ? OR usability_offline = ? OR usability_low_score = ? OR usability_redundant_ip = ? OR usability_gouging = ? OR usability_not_accepting_contracts = ? OR usability_not_announced = ? OR usability_not_completing_scan = ?", - true, true, true, true, true, true, true, true) - case api.UsabilityFilterModeAll: - // nothing to do - } - return db - } -} - func (ss *SQLStore) isBlocked(h dbHost) (blocked bool) { ss.mu.Lock() defer ss.mu.Unlock() diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index c9ab2ba7e..ec3bc17be 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -248,30 +248,163 @@ func TestSearchHosts(t *testing.T) { // add 3 hosts var hks []types.PublicKey - for i := 0; i < 3; i++ { - if err := ss.addCustomTestHost(types.PublicKey{byte(i)}, fmt.Sprintf("-%v-", i+1)); err != nil { + for i := 1; i <= 3; i++ { + if err := ss.addCustomTestHost(types.PublicKey{byte(i)}, fmt.Sprintf("foo.com:100%d", i)); err != nil { t.Fatal(err) } hks = append(hks, types.PublicKey{byte(i)}) } hk1, hk2, hk3 := hks[0], hks[1], hks[2] - // Search by address. - if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "1", nil, 0, -1); err != nil || len(hosts) != 1 { + // search all hosts + his, err := ss.SearchHosts(context.Background(), api.HostFilterModeAll, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 3 { + t.Fatal("unexpected") + } + + // assert offset & limit are taken into account + his, err = ss.SearchHosts(context.Background(), api.HostFilterModeAll, "", nil, 0, 1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } + his, err = ss.SearchHosts(context.Background(), api.HostFilterModeAll, "", nil, 1, 2) + if err != nil { + t.Fatal(err) + } else if len(his) != 2 { + t.Fatal("unexpected") + } + his, err = ss.SearchHosts(context.Background(), api.HostFilterModeAll, "", nil, 3, 1) + if err != nil { + t.Fatal(err) + } else if len(his) != 0 { + t.Fatal("unexpected") + } + + // assert address and key filters are taken into account + if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "com:1001", nil, 0, -1); err != nil || len(hosts) != 1 { t.Fatal("unexpected", len(hosts), err) } - // Filter by key. - if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "", []types.PublicKey{hk1, hk2}, 0, -1); err != nil || len(hosts) != 2 { + if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "", []types.PublicKey{hk2, hk3}, 0, -1); err != nil || len(hosts) != 2 { t.Fatal("unexpected", len(hosts), err) } - // Filter by address and key. - if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "1", []types.PublicKey{hk1, hk2}, 0, -1); err != nil || len(hosts) != 1 { + if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "com:1002", []types.PublicKey{hk2, hk3}, 0, -1); err != nil || len(hosts) != 1 { t.Fatal("unexpected", len(hosts), err) } - // Filter by key and limit results - if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "3", []types.PublicKey{hk3}, 0, -1); err != nil || len(hosts) != 1 { + if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "com:1002", []types.PublicKey{hk1}, 0, -1); err != nil || len(hosts) != 0 { t.Fatal("unexpected", len(hosts), err) } + + // assert host filter mode is taken into account + err = ss.UpdateHostBlocklistEntries(context.Background(), []string{"foo.com:1001"}, nil, false) + if err != nil { + t.Fatal(err) + } + his, err = ss.SearchHosts(context.Background(), api.HostFilterModeAllowed, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 2 { + t.Fatal("unexpected") + } else if his[0].Host.PublicKey != (types.PublicKey{2}) || his[1].Host.PublicKey != (types.PublicKey{3}) { + t.Fatal("unexpected", his[0].Host.PublicKey, his[1].Host.PublicKey) + } + his, err = ss.SearchHosts(context.Background(), api.HostFilterModeBlocked, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } else if his[0].Host.PublicKey != (types.PublicKey{1}) { + t.Fatal("unexpected", his) + } + err = ss.UpdateHostBlocklistEntries(context.Background(), nil, nil, true) + if err != nil { + t.Fatal(err) + } + + // add two autopilots + ap1 := "ap1" + err = ss.UpdateAutopilot(context.Background(), api.Autopilot{ID: ap1}) + if err != nil { + t.Fatal(err) + } + ap2 := "ap2" + err = ss.UpdateAutopilot(context.Background(), api.Autopilot{ID: ap2}) + if err != nil { + t.Fatal(err) + } + + // add host checks, h1 gets ap1 and h2 gets both, h3 gets none + h1c := newTestHostCheck() + h1c.Score.Age = .1 + err = ss.UpdateHostCheck(context.Background(), ap1, hk1, h1c) + if err != nil { + t.Fatal(err) + } + h2c1 := newTestHostCheck() + h2c1.Score.Age = .21 + err = ss.UpdateHostCheck(context.Background(), ap1, hk2, h2c1) + if err != nil { + t.Fatal(err) + } + h2c2 := newTestHostCheck() + h2c2.Score.Age = .22 + err = ss.UpdateHostCheck(context.Background(), ap2, hk2, h2c2) + if err != nil { + t.Fatal(err) + } + + // assert there are currently 3 checks + var cnt int64 + err = ss.db.Model(&dbHostCheck{}).Count(&cnt).Error + if err != nil { + t.Fatal(err) + } else if cnt != 3 { + t.Fatal("unexpected", cnt) + } + + // fetch all hosts + his, err = ss.SearchHosts(context.Background(), api.HostFilterModeAll, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if cnt != 3 { + t.Fatal("unexpected", cnt) + } + + // assert h1 and h2 have the expected checks + if c1, ok := his[0].Checks[ap1]; !ok || c1 != h1c { + t.Fatal("unexpected", c1, ok) + } else if c2, ok := his[1].Checks[ap1]; !ok || c2 != h2c1 { + t.Fatal("unexpected", c2, ok) + } else if c3, ok := his[1].Checks[ap2]; !ok || c3 != h2c2 { + t.Fatal("unexpected", c3, ok) + } + + // assert cascade delete on host + err = ss.db.Exec("DELETE FROM hosts WHERE public_key = ?", publicKey(types.PublicKey{1})).Error + if err != nil { + t.Fatal(err) + } + err = ss.db.Model(&dbHostCheck{}).Count(&cnt).Error + if err != nil { + t.Fatal(err) + } else if cnt != 2 { + t.Fatal("unexpected", cnt) + } + + // assert cascade delete on autopilot + err = ss.db.Exec("DELETE FROM autopilots WHERE identifier IN (?,?)", ap1, ap2).Error + if err != nil { + t.Fatal(err) + } + err = ss.db.Model(&dbHostCheck{}).Count(&cnt).Error + if err != nil { + t.Fatal(err) + } else if cnt != 0 { + t.Fatal("unexpected", cnt) + } } // TestRecordScan is a test for recording scans. @@ -1064,222 +1197,6 @@ func TestAnnouncementMaxAge(t *testing.T) { } } -func TestHostInfo(t *testing.T) { - ss := newTestSQLStore(t, defaultTestSQLStoreConfig) - defer ss.Close() - - // fetch info for a non-existing autopilot - _, err := ss.HostInfo(context.Background(), "foo", types.PublicKey{1}) - if !errors.Is(err, api.ErrAutopilotNotFound) { - t.Fatal(err) - } - - // add autopilot - err = ss.UpdateAutopilot(context.Background(), api.Autopilot{ID: "foo"}) - if err != nil { - t.Fatal(err) - } - - // fetch info for a non-existing host - _, err = ss.HostInfo(context.Background(), "foo", types.PublicKey{1}) - if !errors.Is(err, api.ErrHostNotFound) { - t.Fatal(err) - } - - // add host - err = ss.addTestHost(types.PublicKey{1}) - if err != nil { - t.Fatal(err) - } - h, err := ss.Host(context.Background(), types.PublicKey{1}) - if err != nil { - t.Fatal(err) - } - - // fetch non-existing info - _, err = ss.HostInfo(context.Background(), "foo", types.PublicKey{1}) - if !errors.Is(err, api.ErrHostInfoNotFound) { - t.Fatal(err) - } - - // add host info - want := newTestHostInfo(h.Host) - err = ss.UpdateHostInfo(context.Background(), "foo", types.PublicKey{1}, want.Gouging, want.Score, want.Usability) - if err != nil { - t.Fatal(err) - } - - // fetch info - got, err := ss.HostInfo(context.Background(), "foo", types.PublicKey{1}) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(got, want) { - t.Fatal("mismatch", cmp.Diff(got, want)) - } - - // update info - want.Score.Age = 0 - err = ss.UpdateHostInfo(context.Background(), "foo", types.PublicKey{1}, want.Gouging, want.Score, want.Usability) - if err != nil { - t.Fatal(err) - } - - // fetch info - got, err = ss.HostInfo(context.Background(), "foo", types.PublicKey{1}) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(got, want) { - t.Fatal("mismatch") - } - - // add another host info - err = ss.addCustomTestHost(types.PublicKey{2}, "bar.com:1000") - if err != nil { - t.Fatal(err) - } - err = ss.UpdateHostInfo(context.Background(), "foo", types.PublicKey{2}, want.Gouging, want.Score, want.Usability) - if err != nil { - t.Fatal(err) - } - - // fetch all infos for autopilot - his, err := ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 0, -1) - if err != nil { - t.Fatal(err) - } else if len(his) != 2 { - t.Fatal("unexpected") - } else if his[0].Host.PublicKey != (types.PublicKey{1}) || his[1].Host.PublicKey != (types.PublicKey{2}) { - t.Fatal("unexpected", his) - } - - // fetch infos using offset & limit - his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 0, 1) - if err != nil { - t.Fatal(err) - } else if len(his) != 1 { - t.Fatal("unexpected") - } - his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 1, 1) - if err != nil { - t.Fatal(err) - } else if len(his) != 1 { - t.Fatal("unexpected") - } - his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", nil, 2, 1) - if err != nil { - t.Fatal(err) - } else if len(his) != 0 { - t.Fatal("unexpected") - } - - // fetch infos using net addresses - his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "bar", nil, 0, -1) - if err != nil { - t.Fatal(err) - } else if len(his) != 1 { - t.Fatal("unexpected") - } else if his[0].Host.PublicKey != (types.PublicKey{2}) { - t.Fatal("unexpected", his) - } - - // fetch infos using keyIn - his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeAll, "", []types.PublicKey{{2}}, 0, -1) - if err != nil { - t.Fatal(err) - } else if len(his) != 1 { - t.Fatal("unexpected") - } else if his[0].Host.PublicKey != (types.PublicKey{2}) { - t.Fatal("unexpected", his) - } - - // fetch infos using mode filters - err = ss.UpdateHostBlocklistEntries(context.Background(), []string{"bar.com:1000"}, nil, false) - if err != nil { - t.Fatal(err) - } - his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAllowed, api.UsabilityFilterModeAll, "", nil, 0, -1) - if err != nil { - t.Fatal(err) - } else if len(his) != 1 { - t.Fatal("unexpected") - } else if his[0].Host.PublicKey != (types.PublicKey{1}) { - t.Fatal("unexpected", his) - } - his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeBlocked, api.UsabilityFilterModeAll, "", nil, 0, -1) - if err != nil { - t.Fatal(err) - } else if len(his) != 1 { - t.Fatal("unexpected") - } else if his[0].Host.PublicKey != (types.PublicKey{2}) { - t.Fatal("unexpected", his) - } - err = ss.UpdateHostBlocklistEntries(context.Background(), nil, nil, true) - if err != nil { - t.Fatal(err) - } - - // fetch infos using usability filters - his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeUsable, "", nil, 0, -1) - if err != nil { - t.Fatal(err) - } else if len(his) != 0 { - t.Fatal("unexpected") - } - - // update info - want.Usability.Blocked = false - want.Usability.Offline = false - want.Usability.LowScore = false - want.Usability.RedundantIP = false - want.Usability.Gouging = false - want.Usability.NotAcceptingContracts = false - want.Usability.NotAnnounced = false - want.Usability.NotCompletingScan = false - err = ss.UpdateHostInfo(context.Background(), "foo", types.PublicKey{1}, want.Gouging, want.Score, want.Usability) - if err != nil { - t.Fatal(err) - } - his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeUsable, "", nil, 0, -1) - if err != nil { - t.Fatal(err) - } else if len(his) != 1 { - t.Fatal("unexpected") - } else if his[0].Host.PublicKey != (types.PublicKey{1}) { - t.Fatal("unexpected", his) - } - - // assert cascade delete on host - err = ss.db.Exec("DELETE FROM hosts WHERE public_key = ?", publicKey(types.PublicKey{1})).Error - if err != nil { - t.Fatal(err) - } - his, err = ss.HostInfos(context.Background(), "foo", api.HostFilterModeAll, api.UsabilityFilterModeUsable, "", nil, 0, -1) - if err != nil { - t.Fatal(err) - } else if len(his) != 0 { - t.Fatal("unexpected") - } - - // assert cascade delete on autopilot - var cnt uint64 - err = ss.db.Raw("SELECT COUNT(*) FROM host_infos").Scan(&cnt).Error - if err != nil { - t.Fatal(err) - } else if cnt == 0 { - t.Fatal("unexpected", cnt) - } - err = ss.db.Exec("DELETE FROM autopilots WHERE identifier = ?", "foo").Error - if err != nil { - t.Fatal(err) - } - err = ss.db.Raw("SELECT COUNT(*) FROM host_infos").Scan(&cnt).Error - if err != nil { - t.Fatal(err) - } else if cnt != 0 { - t.Fatal("unexpected", cnt) - } -} - // addTestHosts adds 'n' hosts to the db and returns their keys. func (s *SQLStore) addTestHosts(n int) (keys []types.PublicKey, err error) { cnt, err := s.contractsCount() @@ -1373,9 +1290,9 @@ func newTestTransaction(ha modules.HostAnnouncement, sk types.PrivateKey) stypes return stypes.Transaction{ArbitraryData: [][]byte{buf.Bytes()}} } -func newTestHostInfo(h hostdb.Host) api.HostInfo { - return api.HostInfo{ - Host: h, +func newTestHostCheck() api.HostCheck { + return api.HostCheck{ + Gouging: api.HostGougingBreakdown{ ContractErr: "foo", DownloadErr: "bar", diff --git a/stores/metadata_test.go b/stores/metadata_test.go index c16f927d1..eb082fb54 100644 --- a/stores/metadata_test.go +++ b/stores/metadata_test.go @@ -440,12 +440,12 @@ func TestContractsForHost(t *testing.T) { } contracts, _ := contractsForHost(ss.db, hosts[0]) - if len(contracts) != 1 || contracts[0].Host.convert().PublicKey.String() != hosts[0].convert().PublicKey.String() { + if len(contracts) != 1 || types.PublicKey(contracts[0].Host.PublicKey).String() != types.PublicKey(hosts[0].PublicKey).String() { t.Fatal("unexpected", len(contracts), contracts) } contracts, _ = contractsForHost(ss.db, hosts[1]) - if len(contracts) != 1 || contracts[0].Host.convert().PublicKey.String() != hosts[1].convert().PublicKey.String() { + if len(contracts) != 1 || types.PublicKey(contracts[0].Host.PublicKey).String() != types.PublicKey(hosts[1].PublicKey).String() { t.Fatalf("unexpected contracts, %+v", contracts) } } diff --git a/stores/migrations.go b/stores/migrations.go index 9f874935a..4ac6b755e 100644 --- a/stores/migrations.go +++ b/stores/migrations.go @@ -63,9 +63,9 @@ func performMigrations(db *gorm.DB, logger *zap.SugaredLogger) error { }, }, { - ID: "00007_host_info", + ID: "00007_host_checks", Migrate: func(tx *gorm.DB) error { - return performMigration(tx, dbIdentifier, "00007_host_info", logger) + return performMigration(tx, dbIdentifier, "00007_host_checks", logger) }, }, } diff --git a/stores/migrations/mysql/main/migration_00007_host_checks.sql b/stores/migrations/mysql/main/migration_00007_host_checks.sql new file mode 100644 index 000000000..f96b9853c --- /dev/null +++ b/stores/migrations/mysql/main/migration_00007_host_checks.sql @@ -0,0 +1,52 @@ +-- dbHostCheck +CREATE TABLE `host_checks` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `created_at` datetime(3) DEFAULT NULL, + + `db_autopilot_id` bigint unsigned NOT NULL, + `db_host_id` bigint unsigned NOT NULL, + + `usability_blocked` boolean NOT NULL DEFAULT false, + `usability_offline` boolean NOT NULL DEFAULT false, + `usability_low_score` boolean NOT NULL DEFAULT false, + `usability_redundant_ip` boolean NOT NULL DEFAULT false, + `usability_gouging` boolean NOT NULL DEFAULT false, + `usability_not_accepting_contracts` boolean NOT NULL DEFAULT false, + `usability_not_announced` boolean NOT NULL DEFAULT false, + `usability_not_completing_scan` boolean NOT NULL DEFAULT false, + + `score_age` double NOT NULL, + `score_collateral` double NOT NULL, + `score_interactions` double NOT NULL, + `score_storage_remaining` double NOT NULL, + `score_uptime` double NOT NULL, + `score_version` double NOT NULL, + `score_prices` double NOT NULL, + + `gouging_contract_err` text, + `gouging_download_err` text, + `gouging_gouging_err` text, + `gouging_prune_err` text, + `gouging_upload_err` text, + + PRIMARY KEY (`id`), + UNIQUE KEY `idx_host_checks_id` (`db_autopilot_id`, `db_host_id`), + INDEX `idx_host_checks_usability_blocked` (`usability_blocked`), + INDEX `idx_host_checks_usability_offline` (`usability_offline`), + INDEX `idx_host_checks_usability_low_score` (`usability_low_score`), + INDEX `idx_host_checks_usability_redundant_ip` (`usability_redundant_ip`), + INDEX `idx_host_checks_usability_gouging` (`usability_gouging`), + INDEX `idx_host_checks_usability_not_accepting_contracts` (`usability_not_accepting_contracts`), + INDEX `idx_host_checks_usability_not_announced` (`usability_not_announced`), + INDEX `idx_host_checks_usability_not_completing_scan` (`usability_not_completing_scan`), + INDEX `idx_host_checks_score_age` (`score_age`), + INDEX `idx_host_checks_score_collateral` (`score_collateral`), + INDEX `idx_host_checks_score_interactions` (`score_interactions`), + INDEX `idx_host_checks_score_storage_remaining` (`score_storage_remaining`), + INDEX `idx_host_checks_score_uptime` (`score_uptime`), + INDEX `idx_host_checks_score_version` (`score_version`), + INDEX `idx_host_checks_score_prices` (`score_prices`), + + CONSTRAINT `fk_host_checks_autopilot` FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_host_checks_host` FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; diff --git a/stores/migrations/mysql/main/migration_00007_host_info.sql b/stores/migrations/mysql/main/migration_00007_host_info.sql deleted file mode 100644 index c13f5c396..000000000 --- a/stores/migrations/mysql/main/migration_00007_host_info.sql +++ /dev/null @@ -1,52 +0,0 @@ --- dbHostInfo -CREATE TABLE `host_infos` ( - `id` bigint unsigned NOT NULL AUTO_INCREMENT, - `created_at` datetime(3) DEFAULT NULL, - - `db_autopilot_id` bigint unsigned NOT NULL, - `db_host_id` bigint unsigned NOT NULL, - - `usability_blocked` boolean NOT NULL DEFAULT false, - `usability_offline` boolean NOT NULL DEFAULT false, - `usability_low_score` boolean NOT NULL DEFAULT false, - `usability_redundant_ip` boolean NOT NULL DEFAULT false, - `usability_gouging` boolean NOT NULL DEFAULT false, - `usability_not_accepting_contracts` boolean NOT NULL DEFAULT false, - `usability_not_announced` boolean NOT NULL DEFAULT false, - `usability_not_completing_scan` boolean NOT NULL DEFAULT false, - - `score_age` double NOT NULL, - `score_collateral` double NOT NULL, - `score_interactions` double NOT NULL, - `score_storage_remaining` double NOT NULL, - `score_uptime` double NOT NULL, - `score_version` double NOT NULL, - `score_prices` double NOT NULL, - - `gouging_contract_err` text, - `gouging_download_err` text, - `gouging_gouging_err` text, - `gouging_prune_err` text, - `gouging_upload_err` text, - - PRIMARY KEY (`id`), - UNIQUE KEY `idx_host_infos_id` (`db_autopilot_id`, `db_host_id`), - INDEX `idx_host_infos_usability_blocked` (`usability_blocked`), - INDEX `idx_host_infos_usability_offline` (`usability_offline`), - INDEX `idx_host_infos_usability_low_score` (`usability_low_score`), - INDEX `idx_host_infos_usability_redundant_ip` (`usability_redundant_ip`), - INDEX `idx_host_infos_usability_gouging` (`usability_gouging`), - INDEX `idx_host_infos_usability_not_accepting_contracts` (`usability_not_accepting_contracts`), - INDEX `idx_host_infos_usability_not_announced` (`usability_not_announced`), - INDEX `idx_host_infos_usability_not_completing_scan` (`usability_not_completing_scan`), - INDEX `idx_host_infos_score_age` (`score_age`), - INDEX `idx_host_infos_score_collateral` (`score_collateral`), - INDEX `idx_host_infos_score_interactions` (`score_interactions`), - INDEX `idx_host_infos_score_storage_remaining` (`score_storage_remaining`), - INDEX `idx_host_infos_score_uptime` (`score_uptime`), - INDEX `idx_host_infos_score_version` (`score_version`), - INDEX `idx_host_infos_score_prices` (`score_prices`), - - CONSTRAINT `fk_host_infos_autopilot` FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, - CONSTRAINT `fk_host_infos_host` FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; diff --git a/stores/migrations/mysql/main/schema.sql b/stores/migrations/mysql/main/schema.sql index e39b7f963..446b2a805 100644 --- a/stores/migrations/mysql/main/schema.sql +++ b/stores/migrations/mysql/main/schema.sql @@ -422,8 +422,8 @@ CREATE TABLE `object_user_metadata` ( CONSTRAINT `fk_multipart_upload_user_metadata` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads` (`id`) ON DELETE SET NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; --- dbHostInfo -CREATE TABLE `host_infos` ( +-- dbHostCheck +CREATE TABLE `host_checks` ( `id` bigint unsigned NOT NULL AUTO_INCREMENT, `created_at` datetime(3) DEFAULT NULL, @@ -454,25 +454,25 @@ CREATE TABLE `host_infos` ( `gouging_upload_err` text, PRIMARY KEY (`id`), - UNIQUE KEY `idx_host_infos_id` (`db_autopilot_id`, `db_host_id`), - INDEX `idx_host_infos_usability_blocked` (`usability_blocked`), - INDEX `idx_host_infos_usability_offline` (`usability_offline`), - INDEX `idx_host_infos_usability_low_score` (`usability_low_score`), - INDEX `idx_host_infos_usability_redundant_ip` (`usability_redundant_ip`), - INDEX `idx_host_infos_usability_gouging` (`usability_gouging`), - INDEX `idx_host_infos_usability_not_accepting_contracts` (`usability_not_accepting_contracts`), - INDEX `idx_host_infos_usability_not_announced` (`usability_not_announced`), - INDEX `idx_host_infos_usability_not_completing_scan` (`usability_not_completing_scan`), - INDEX `idx_host_infos_score_age` (`score_age`), - INDEX `idx_host_infos_score_collateral` (`score_collateral`), - INDEX `idx_host_infos_score_interactions` (`score_interactions`), - INDEX `idx_host_infos_score_storage_remaining` (`score_storage_remaining`), - INDEX `idx_host_infos_score_uptime` (`score_uptime`), - INDEX `idx_host_infos_score_version` (`score_version`), - INDEX `idx_host_infos_score_prices` (`score_prices`), - - CONSTRAINT `fk_host_infos_autopilot` FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, - CONSTRAINT `fk_host_infos_host` FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE + UNIQUE KEY `idx_host_checks_id` (`db_autopilot_id`, `db_host_id`), + INDEX `idx_host_checks_usability_blocked` (`usability_blocked`), + INDEX `idx_host_checks_usability_offline` (`usability_offline`), + INDEX `idx_host_checks_usability_low_score` (`usability_low_score`), + INDEX `idx_host_checks_usability_redundant_ip` (`usability_redundant_ip`), + INDEX `idx_host_checks_usability_gouging` (`usability_gouging`), + INDEX `idx_host_checks_usability_not_accepting_contracts` (`usability_not_accepting_contracts`), + INDEX `idx_host_checks_usability_not_announced` (`usability_not_announced`), + INDEX `idx_host_checks_usability_not_completing_scan` (`usability_not_completing_scan`), + INDEX `idx_host_checks_score_age` (`score_age`), + INDEX `idx_host_checks_score_collateral` (`score_collateral`), + INDEX `idx_host_checks_score_interactions` (`score_interactions`), + INDEX `idx_host_checks_score_storage_remaining` (`score_storage_remaining`), + INDEX `idx_host_checks_score_uptime` (`score_uptime`), + INDEX `idx_host_checks_score_version` (`score_version`), + INDEX `idx_host_checks_score_prices` (`score_prices`), + + CONSTRAINT `fk_host_checks_autopilot` FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_host_checks_host` FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; -- create default bucket diff --git a/stores/migrations/sqlite/main/migration_00007_host_checks.sql b/stores/migrations/sqlite/main/migration_00007_host_checks.sql new file mode 100644 index 000000000..da13460c6 --- /dev/null +++ b/stores/migrations/sqlite/main/migration_00007_host_checks.sql @@ -0,0 +1,52 @@ +-- dbHostCheck +CREATE TABLE `host_checks` ( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `created_at` datetime, + + `db_autopilot_id` INTEGER NOT NULL, + `db_host_id` INTEGER NOT NULL, + + `usability_blocked` INTEGER NOT NULL DEFAULT 0, + `usability_offline` INTEGER NOT NULL DEFAULT 0, + `usability_low_score` INTEGER NOT NULL DEFAULT 0, + `usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, + `usability_gouging` INTEGER NOT NULL DEFAULT 0, + `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, + `usability_not_announced` INTEGER NOT NULL DEFAULT 0, + `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, + + `score_age` REAL NOT NULL, + `score_collateral` REAL NOT NULL, + `score_interactions` REAL NOT NULL, + `score_storage_remaining` REAL NOT NULL, + `score_uptime` REAL NOT NULL, + `score_version` REAL NOT NULL, + `score_prices` REAL NOT NULL, + + `gouging_contract_err` TEXT, + `gouging_download_err` TEXT, + `gouging_gouging_err` TEXT, + `gouging_prune_err` TEXT, + `gouging_upload_err` TEXT, + + FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, + FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE +); + +-- Indexes creation +CREATE UNIQUE INDEX `idx_host_checks_id` ON `host_checks` (`db_autopilot_id`, `db_host_id`); +CREATE INDEX `idx_host_checks_usability_blocked` ON `host_checks` (`usability_blocked`); +CREATE INDEX `idx_host_checks_usability_offline` ON `host_checks` (`usability_offline`); +CREATE INDEX `idx_host_checks_usability_low_score` ON `host_checks` (`usability_low_score`); +CREATE INDEX `idx_host_checks_usability_redundant_ip` ON `host_checks` (`usability_redundant_ip`); +CREATE INDEX `idx_host_checks_usability_gouging` ON `host_checks` (`usability_gouging`); +CREATE INDEX `idx_host_checks_usability_not_accepting_contracts` ON `host_checks` (`usability_not_accepting_contracts`); +CREATE INDEX `idx_host_checks_usability_not_announced` ON `host_checks` (`usability_not_announced`); +CREATE INDEX `idx_host_checks_usability_not_completing_scan` ON `host_checks` (`usability_not_completing_scan`); +CREATE INDEX `idx_host_checks_score_age` ON `host_checks` (`score_age`); +CREATE INDEX `idx_host_checks_score_collateral` ON `host_checks` (`score_collateral`); +CREATE INDEX `idx_host_checks_score_interactions` ON `host_checks` (`score_interactions`); +CREATE INDEX `idx_host_checks_score_storage_remaining` ON `host_checks` (`score_storage_remaining`); +CREATE INDEX `idx_host_checks_score_uptime` ON `host_checks` (`score_uptime`); +CREATE INDEX `idx_host_checks_score_version` ON `host_checks` (`score_version`); +CREATE INDEX `idx_host_checks_score_prices` ON `host_checks` (`score_prices`); diff --git a/stores/migrations/sqlite/main/migration_00007_host_info.sql b/stores/migrations/sqlite/main/migration_00007_host_info.sql deleted file mode 100644 index 910dd637c..000000000 --- a/stores/migrations/sqlite/main/migration_00007_host_info.sql +++ /dev/null @@ -1,52 +0,0 @@ --- dbHostInfo -CREATE TABLE `host_infos` ( - `id` INTEGER PRIMARY KEY AUTOINCREMENT, - `created_at` datetime, - - `db_autopilot_id` INTEGER NOT NULL, - `db_host_id` INTEGER NOT NULL, - - `usability_blocked` INTEGER NOT NULL DEFAULT 0, - `usability_offline` INTEGER NOT NULL DEFAULT 0, - `usability_low_score` INTEGER NOT NULL DEFAULT 0, - `usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, - `usability_gouging` INTEGER NOT NULL DEFAULT 0, - `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, - `usability_not_announced` INTEGER NOT NULL DEFAULT 0, - `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, - - `score_age` REAL NOT NULL, - `score_collateral` REAL NOT NULL, - `score_interactions` REAL NOT NULL, - `score_storage_remaining` REAL NOT NULL, - `score_uptime` REAL NOT NULL, - `score_version` REAL NOT NULL, - `score_prices` REAL NOT NULL, - - `gouging_contract_err` TEXT, - `gouging_download_err` TEXT, - `gouging_gouging_err` TEXT, - `gouging_prune_err` TEXT, - `gouging_upload_err` TEXT, - - FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, - FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE -); - --- Indexes creation -CREATE UNIQUE INDEX `idx_host_infos_id` ON `host_infos` (`db_autopilot_id`, `db_host_id`); -CREATE INDEX `idx_host_infos_usability_blocked` ON `host_infos` (`usability_blocked`); -CREATE INDEX `idx_host_infos_usability_offline` ON `host_infos` (`usability_offline`); -CREATE INDEX `idx_host_infos_usability_low_score` ON `host_infos` (`usability_low_score`); -CREATE INDEX `idx_host_infos_usability_redundant_ip` ON `host_infos` (`usability_redundant_ip`); -CREATE INDEX `idx_host_infos_usability_gouging` ON `host_infos` (`usability_gouging`); -CREATE INDEX `idx_host_infos_usability_not_accepting_contracts` ON `host_infos` (`usability_not_accepting_contracts`); -CREATE INDEX `idx_host_infos_usability_not_announced` ON `host_infos` (`usability_not_announced`); -CREATE INDEX `idx_host_infos_usability_not_completing_scan` ON `host_infos` (`usability_not_completing_scan`); -CREATE INDEX `idx_host_infos_score_age` ON `host_infos` (`score_age`); -CREATE INDEX `idx_host_infos_score_collateral` ON `host_infos` (`score_collateral`); -CREATE INDEX `idx_host_infos_score_interactions` ON `host_infos` (`score_interactions`); -CREATE INDEX `idx_host_infos_score_storage_remaining` ON `host_infos` (`score_storage_remaining`); -CREATE INDEX `idx_host_infos_score_uptime` ON `host_infos` (`score_uptime`); -CREATE INDEX `idx_host_infos_score_version` ON `host_infos` (`score_version`); -CREATE INDEX `idx_host_infos_score_prices` ON `host_infos` (`score_prices`); diff --git a/stores/migrations/sqlite/main/schema.sql b/stores/migrations/sqlite/main/schema.sql index 791fce1ca..3fca53a3a 100644 --- a/stores/migrations/sqlite/main/schema.sql +++ b/stores/migrations/sqlite/main/schema.sql @@ -149,24 +149,24 @@ CREATE UNIQUE INDEX `idx_module_event_url` ON `webhooks`(`module`,`event`,`url`) CREATE TABLE `object_user_metadata` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`db_object_id` integer DEFAULT NULL,`db_multipart_upload_id` integer DEFAULT NULL,`key` text NOT NULL,`value` text, CONSTRAINT `fk_object_user_metadata` FOREIGN KEY (`db_object_id`) REFERENCES `objects` (`id`) ON DELETE CASCADE, CONSTRAINT `fk_multipart_upload_user_metadata` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads` (`id`) ON DELETE SET NULL); CREATE UNIQUE INDEX `idx_object_user_metadata_key` ON `object_user_metadata`(`db_object_id`,`db_multipart_upload_id`,`key`); --- dbHostInfo -CREATE TABLE `host_infos` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `created_at` datetime, `db_autopilot_id` INTEGER NOT NULL, `db_host_id` INTEGER NOT NULL, `usability_blocked` INTEGER NOT NULL DEFAULT 0, `usability_offline` INTEGER NOT NULL DEFAULT 0, `usability_low_score` INTEGER NOT NULL DEFAULT 0, `usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, `usability_gouging` INTEGER NOT NULL DEFAULT 0, `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, `usability_not_announced` INTEGER NOT NULL DEFAULT 0, `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, `score_age` REAL NOT NULL, `score_collateral` REAL NOT NULL, `score_interactions` REAL NOT NULL, `score_storage_remaining` REAL NOT NULL, `score_uptime` REAL NOT NULL, `score_version` REAL NOT NULL, `score_prices` REAL NOT NULL, `gouging_contract_err` TEXT, `gouging_download_err` TEXT, `gouging_gouging_err` TEXT, `gouging_prune_err` TEXT, `gouging_upload_err` TEXT, FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE); -CREATE UNIQUE INDEX `idx_host_infos_id` ON `host_infos` (`db_autopilot_id`, `db_host_id`); -CREATE INDEX `idx_host_infos_usability_blocked` ON `host_infos` (`usability_blocked`); -CREATE INDEX `idx_host_infos_usability_offline` ON `host_infos` (`usability_offline`); -CREATE INDEX `idx_host_infos_usability_low_score` ON `host_infos` (`usability_low_score`); -CREATE INDEX `idx_host_infos_usability_redundant_ip` ON `host_infos` (`usability_redundant_ip`); -CREATE INDEX `idx_host_infos_usability_gouging` ON `host_infos` (`usability_gouging`); -CREATE INDEX `idx_host_infos_usability_not_accepting_contracts` ON `host_infos` (`usability_not_accepting_contracts`); -CREATE INDEX `idx_host_infos_usability_not_announced` ON `host_infos` (`usability_not_announced`); -CREATE INDEX `idx_host_infos_usability_not_completing_scan` ON `host_infos` (`usability_not_completing_scan`); -CREATE INDEX `idx_host_infos_score_age` ON `host_infos` (`score_age`); -CREATE INDEX `idx_host_infos_score_collateral` ON `host_infos` (`score_collateral`); -CREATE INDEX `idx_host_infos_score_interactions` ON `host_infos` (`score_interactions`); -CREATE INDEX `idx_host_infos_score_storage_remaining` ON `host_infos` (`score_storage_remaining`); -CREATE INDEX `idx_host_infos_score_uptime` ON `host_infos` (`score_uptime`); -CREATE INDEX `idx_host_infos_score_version` ON `host_infos` (`score_version`); -CREATE INDEX `idx_host_infos_score_prices` ON `host_infos` (`score_prices`); +-- dbHostCheck +CREATE TABLE `host_checks` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `created_at` datetime, `db_autopilot_id` INTEGER NOT NULL, `db_host_id` INTEGER NOT NULL, `usability_blocked` INTEGER NOT NULL DEFAULT 0, `usability_offline` INTEGER NOT NULL DEFAULT 0, `usability_low_score` INTEGER NOT NULL DEFAULT 0, `usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, `usability_gouging` INTEGER NOT NULL DEFAULT 0, `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, `usability_not_announced` INTEGER NOT NULL DEFAULT 0, `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, `score_age` REAL NOT NULL, `score_collateral` REAL NOT NULL, `score_interactions` REAL NOT NULL, `score_storage_remaining` REAL NOT NULL, `score_uptime` REAL NOT NULL, `score_version` REAL NOT NULL, `score_prices` REAL NOT NULL, `gouging_contract_err` TEXT, `gouging_download_err` TEXT, `gouging_gouging_err` TEXT, `gouging_prune_err` TEXT, `gouging_upload_err` TEXT, FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE); +CREATE UNIQUE INDEX `idx_host_checks_id` ON `host_checks` (`db_autopilot_id`, `db_host_id`); +CREATE INDEX `idx_host_checks_usability_blocked` ON `host_checks` (`usability_blocked`); +CREATE INDEX `idx_host_checks_usability_offline` ON `host_checks` (`usability_offline`); +CREATE INDEX `idx_host_checks_usability_low_score` ON `host_checks` (`usability_low_score`); +CREATE INDEX `idx_host_checks_usability_redundant_ip` ON `host_checks` (`usability_redundant_ip`); +CREATE INDEX `idx_host_checks_usability_gouging` ON `host_checks` (`usability_gouging`); +CREATE INDEX `idx_host_checks_usability_not_accepting_contracts` ON `host_checks` (`usability_not_accepting_contracts`); +CREATE INDEX `idx_host_checks_usability_not_announced` ON `host_checks` (`usability_not_announced`); +CREATE INDEX `idx_host_checks_usability_not_completing_scan` ON `host_checks` (`usability_not_completing_scan`); +CREATE INDEX `idx_host_checks_score_age` ON `host_checks` (`score_age`); +CREATE INDEX `idx_host_checks_score_collateral` ON `host_checks` (`score_collateral`); +CREATE INDEX `idx_host_checks_score_interactions` ON `host_checks` (`score_interactions`); +CREATE INDEX `idx_host_checks_score_storage_remaining` ON `host_checks` (`score_storage_remaining`); +CREATE INDEX `idx_host_checks_score_uptime` ON `host_checks` (`score_uptime`); +CREATE INDEX `idx_host_checks_score_version` ON `host_checks` (`score_version`); +CREATE INDEX `idx_host_checks_score_prices` ON `host_checks` (`score_prices`); -- create default bucket INSERT INTO buckets (created_at, name) VALUES (CURRENT_TIMESTAMP, 'default'); diff --git a/worker/mocks_test.go b/worker/mocks_test.go index 7b3609c0b..6e324f67e 100644 --- a/worker/mocks_test.go +++ b/worker/mocks_test.go @@ -260,13 +260,20 @@ var errSectorOutOfBounds = errors.New("sector out of bounds") type hostMock struct { hk types.PublicKey - hi hostdb.HostInfo + hi api.HostInfo } func newHostMock(hk types.PublicKey) *hostMock { return &hostMock{ hk: hk, - hi: hostdb.HostInfo{Host: hostdb.Host{PublicKey: hk, Scanned: true}}, + hi: api.HostInfo{ + HostInfo: hostdb.HostInfo{ + Host: hostdb.Host{ + PublicKey: hk, + Scanned: true, + }, + }, + }, } } @@ -282,13 +289,13 @@ func newHostStoreMock() *hostStoreMock { return &hostStoreMock{hosts: make(map[types.PublicKey]*hostMock)} } -func (hs *hostStoreMock) Host(ctx context.Context, hostKey types.PublicKey) (hostdb.HostInfo, error) { +func (hs *hostStoreMock) Host(ctx context.Context, hostKey types.PublicKey) (api.HostInfo, error) { hs.mu.Lock() defer hs.mu.Unlock() h, ok := hs.hosts[hostKey] if !ok { - return hostdb.HostInfo{}, api.ErrHostNotFound + return api.HostInfo{}, api.ErrHostNotFound } return h.hi, nil } diff --git a/worker/worker.go b/worker/worker.go index 707a6a7f1..fda44dee6 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -110,7 +110,7 @@ type ( RecordPriceTables(ctx context.Context, priceTableUpdate []hostdb.PriceTableUpdate) error RecordContractSpending(ctx context.Context, records []api.ContractSpendingRecord) error - Host(ctx context.Context, hostKey types.PublicKey) (hostdb.HostInfo, error) + Host(ctx context.Context, hostKey types.PublicKey) (api.HostInfo, error) } ObjectStore interface { From 949e6c35477957f44843f2fa8f5323706f1c46a3 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 21 Mar 2024 20:42:07 +0100 Subject: [PATCH 05/16] all: cleanup PR --- api/host.go | 10 +++--- autopilot/autopilot.go | 10 +++--- autopilot/autopilot_test.go | 4 +-- autopilot/contractor.go | 6 ++-- autopilot/hostinfo.go | 2 +- autopilot/scanner.go | 2 +- autopilot/scanner_test.go | 6 ++-- bus/bus.go | 4 +-- bus/client/hosts.go | 4 +-- stores/hostdb.go | 66 ++++++++++--------------------------- worker/mocks_test.go | 8 ++--- worker/worker.go | 2 +- 12 files changed, 46 insertions(+), 78 deletions(-) diff --git a/api/host.go b/api/host.go index 50b058642..6c95165c9 100644 --- a/api/host.go +++ b/api/host.go @@ -24,10 +24,6 @@ var ( // ErrHostNotFound is returned when a host can't be retrieved from the // database. ErrHostNotFound = errors.New("host doesn't exist in hostdb") - - // ErrHostInfoNotFound is returned when host info can't be retrieved from - // the database. - ErrHostInfoNotFound = errors.New("host info doesn't exist in hostdb") ) type ( @@ -47,11 +43,14 @@ type ( MinRecentScanFailures uint64 `json:"minRecentScanFailures"` } + // HostsRequest is the request type for the /api/autopilot/hosts endpoint. HostsRequest struct { UsabilityMode string `json:"usabilityMode"` SearchHostsRequest } + // SearchHostsRequest is the request type for the /api/bus/search/hosts + // endpoint. SearchHostsRequest struct { Offset int `json:"offset"` Limit int `json:"limit"` @@ -92,7 +91,6 @@ type ( SearchHostOptions struct { AddressContains string FilterMode string - UsabilityMode string KeyIn []types.PublicKey Limit int Offset int @@ -121,7 +119,7 @@ func (opts HostsForScanningOptions) Apply(values url.Values) { } type ( - HostInfo struct { + Host struct { hostdb.HostInfo Checks map[string]HostCheck `json:"checks"` } diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 7562de960..b7b2f97d4 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -53,10 +53,10 @@ type Bus interface { PrunableData(ctx context.Context) (prunableData api.ContractsPrunableDataResponse, err error) // hostdb - Host(ctx context.Context, hostKey types.PublicKey) (api.HostInfo, error) + Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error) RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) - SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.HostInfo, error) + SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.Host, error) // metrics RecordContractSetChurnMetric(ctx context.Context, metrics ...api.ContractSetChurnMetric) error @@ -737,7 +737,7 @@ func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { jc.Encode(hosts) } -func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.HostInfo) (usables uint64) { +func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.Host) (usables uint64) { gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow) for _, host := range hosts { usable, _ := isUsableHost(cfg, rs, gc, host.HostInfo, smallestValidScore, 0) @@ -751,7 +751,7 @@ func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types. // evaluateConfig evaluates the given configuration and if the gouging settings // are too strict for the number of contracts required by 'cfg', it will provide // a recommendation on how to loosen it. -func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.HostInfo) (resp api.ConfigEvaluationResponse) { +func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.Host) (resp api.ConfigEvaluationResponse) { gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow) resp.Hosts = uint64(len(hosts)) @@ -866,7 +866,7 @@ func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Cu // optimiseGougingSetting tries to optimise one field of the gouging settings to // try and hit the target number of contracts. -func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, hosts []api.HostInfo) bool { +func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, hosts []api.Host) bool { if cfg.Contracts.Amount == 0 { return true // nothing to do } diff --git a/autopilot/autopilot_test.go b/autopilot/autopilot_test.go index 2da3fc7be..2edf88516 100644 --- a/autopilot/autopilot_test.go +++ b/autopilot/autopilot_test.go @@ -14,9 +14,9 @@ import ( func TestOptimiseGougingSetting(t *testing.T) { // create 10 hosts that should all be usable - var hosts []api.HostInfo + var hosts []api.Host for i := 0; i < 10; i++ { - hosts = append(hosts, api.HostInfo{ + hosts = append(hosts, api.Host{ HostInfo: hostdb.HostInfo{ Host: hostdb.Host{ KnownSince: time.Unix(0, 0), diff --git a/autopilot/contractor.go b/autopilot/contractor.go index d376d682e..6f530d405 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -1297,7 +1297,7 @@ func (c *contractor) calculateMinScore(candidates []scoredHost, numContracts uin return minScore } -func (c *contractor) candidateHosts(ctx context.Context, hosts []api.HostInfo, usedHosts map[types.PublicKey]struct{}, storedData map[types.PublicKey]uint64, minScore float64) ([]scoredHost, unusableHostResult, error) { +func (c *contractor) candidateHosts(ctx context.Context, hosts []api.Host, usedHosts map[types.PublicKey]struct{}, storedData map[types.PublicKey]uint64, minScore float64) ([]scoredHost, unusableHostResult, error) { start := time.Now() // fetch consensus state @@ -1311,7 +1311,7 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []api.HostInfo, u gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) // select unused hosts that passed a scan - var unused []api.HostInfo + var unused []api.Host var excluded, notcompletedscan int for _, h := range hosts { // filter out used hosts @@ -1612,7 +1612,7 @@ func (c *contractor) tryPerformPruning(wp *workerPool) { }() } -func (c *contractor) hostForContract(ctx context.Context, fcid types.FileContractID) (host api.HostInfo, metadata api.ContractMetadata, err error) { +func (c *contractor) hostForContract(ctx context.Context, fcid types.FileContractID) (host api.Host, metadata api.ContractMetadata, err error) { // fetch the contract metadata, err = c.ap.bus.Contract(ctx, fcid) if err != nil { diff --git a/autopilot/hostinfo.go b/autopilot/hostinfo.go index 5af554e6b..d48ce7760 100644 --- a/autopilot/hostinfo.go +++ b/autopilot/hostinfo.go @@ -66,7 +66,7 @@ func (c *contractor) HostInfo(ctx context.Context, hostKey types.PublicKey) (api }, nil } -func (c *contractor) hostInfoFromCache(ctx context.Context, host api.HostInfo) (hi hostInfo, found bool) { +func (c *contractor) hostInfoFromCache(ctx context.Context, host api.Host) (hi hostInfo, found bool) { // grab host details from cache c.mu.Lock() hi, found = c.cachedHostInfo[host.PublicKey] diff --git a/autopilot/scanner.go b/autopilot/scanner.go index a2d30abfa..28b2e1fe9 100644 --- a/autopilot/scanner.go +++ b/autopilot/scanner.go @@ -31,7 +31,7 @@ type ( // a bit, we currently use inline interfaces to avoid having to update the // scanner tests with every interface change bus interface { - SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.HostInfo, error) + SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.Host, error) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error) RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) } diff --git a/autopilot/scanner_test.go b/autopilot/scanner_test.go index 027366662..435e03b90 100644 --- a/autopilot/scanner_test.go +++ b/autopilot/scanner_test.go @@ -19,7 +19,7 @@ type mockBus struct { reqs []string } -func (b *mockBus) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.HostInfo, error) { +func (b *mockBus) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.Host, error) { b.reqs = append(b.reqs, fmt.Sprintf("%d-%d", opts.Offset, opts.Offset+opts.Limit)) start := opts.Offset @@ -32,9 +32,9 @@ func (b *mockBus) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ( end = len(b.hosts) } - his := make([]api.HostInfo, len(b.hosts[start:end])) + his := make([]api.Host, len(b.hosts[start:end])) for i, h := range b.hosts[start:end] { - his[i] = api.HostInfo{ + his[i] = api.Host{ HostInfo: hostdb.HostInfo{ Host: h, }, diff --git a/bus/bus.go b/bus/bus.go index 510353edd..d68e46309 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -91,13 +91,13 @@ type ( // A HostDB stores information about hosts. HostDB interface { - Host(ctx context.Context, hostKey types.PublicKey) (api.HostInfo, error) + Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]hostdb.HostAddress, error) RecordHostScans(ctx context.Context, scans []hostdb.HostScan) error RecordPriceTables(ctx context.Context, priceTableUpdate []hostdb.PriceTableUpdate) error RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) ResetLostSectors(ctx context.Context, hk types.PublicKey) error - SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.HostInfo, error) + SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.Host, error) HostAllowlist(ctx context.Context) ([]types.PublicKey, error) HostBlocklist(ctx context.Context) ([]string, error) diff --git a/bus/client/hosts.go b/bus/client/hosts.go index 460291dd6..f0ab56fd1 100644 --- a/bus/client/hosts.go +++ b/bus/client/hosts.go @@ -12,7 +12,7 @@ import ( ) // Host returns information about a particular host known to the server. -func (c *Client) Host(ctx context.Context, hostKey types.PublicKey) (h api.HostInfo, err error) { +func (c *Client) Host(ctx context.Context, hostKey types.PublicKey) (h api.Host, err error) { err = c.c.WithContext(ctx).GET(fmt.Sprintf("/host/%s", hostKey), &h) return } @@ -78,7 +78,7 @@ func (c *Client) ResetLostSectors(ctx context.Context, hostKey types.PublicKey) } // SearchHosts returns all hosts that match certain search criteria. -func (c *Client) SearchHosts(ctx context.Context, opts api.SearchHostOptions) (hosts []api.HostInfo, err error) { +func (c *Client) SearchHosts(ctx context.Context, opts api.SearchHostOptions) (hosts []api.Host, err error) { err = c.c.WithContext(ctx).POST("/search/hosts", api.SearchHostsRequest{ Offset: opts.Offset, Limit: opts.Limit, diff --git a/stores/hostdb.go b/stores/hostdb.go index 01a9e594e..f97af5fe9 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -312,7 +312,7 @@ func (dbAllowlistEntry) TableName() string { return "host_allowlist_entries" } func (dbBlocklistEntry) TableName() string { return "host_blocklist_entries" } // convert converts a host into a api.HostInfo -func (h dbHost) convert(blocked bool) api.HostInfo { +func (h dbHost) convert(blocked bool) api.Host { var lastScan time.Time if h.LastScan > 0 { lastScan = time.Unix(0, h.LastScan) @@ -321,7 +321,7 @@ func (h dbHost) convert(blocked bool) api.HostInfo { for _, check := range h.Checks { checks[check.DBAutopilot.Identifier] = check.convert() } - return api.HostInfo{ + return api.Host{ HostInfo: hostdb.HostInfo{ Host: hostdb.Host{ KnownSince: h.CreatedAt, @@ -488,7 +488,7 @@ func (e *dbBlocklistEntry) blocks(h dbHost) bool { } // Host returns information about a host. -func (ss *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (api.HostInfo, error) { +func (ss *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) { var h dbHost tx := ss.db. @@ -498,47 +498,14 @@ func (ss *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (api.Host Preload("Blocklist"). Take(&h) if errors.Is(tx.Error, gorm.ErrRecordNotFound) { - return api.HostInfo{}, api.ErrHostNotFound + return api.Host{}, api.ErrHostNotFound } else if tx.Error != nil { - return api.HostInfo{}, tx.Error + return api.Host{}, tx.Error } return h.convert(ss.isBlocked(h)), nil } -func (ss *SQLStore) HostInfo(ctx context.Context, autopilotID string, hk types.PublicKey) (hi api.HostInfo, err error) { - err = ss.db.Transaction(func(tx *gorm.DB) error { - var entity dbHostCheck - if err := tx. - Model(&dbHostCheck{}). - Where("db_autopilot_id = (?)", gorm.Expr("SELECT id FROM autopilots WHERE identifier = ?", autopilotID)). - Where("db_host_id = (?)", gorm.Expr("SELECT id FROM hosts WHERE public_key = ?", publicKey(hk))). - Preload("DBHost"). - First(&entity). - Error; errors.Is(err, gorm.ErrRecordNotFound) { - if err := tx. - Model(&dbAutopilot{}). - Where("identifier = ?", autopilotID). - First(nil). - Error; errors.Is(err, gorm.ErrRecordNotFound) { - return api.ErrAutopilotNotFound - } else if err := tx. - Model(&dbHost{}). - Where("public_key = ?", publicKey(hk)). - First(nil). - Error; errors.Is(err, gorm.ErrRecordNotFound) { - return api.ErrHostNotFound - } - return api.ErrHostInfoNotFound - } else if err != nil { - return err - } - // hi = entity.convert() - return nil - }) - return -} - func (ss *SQLStore) UpdateHostCheck(ctx context.Context, autopilotID string, hk types.PublicKey, hc api.HostCheck) (err error) { err = ss.db.Transaction(func(tx *gorm.DB) error { // fetch ap id @@ -640,7 +607,7 @@ func (ss *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time, return hostAddresses, err } -func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.HostInfo, error) { +func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.Host, error) { if offset < 0 { return nil, ErrNegativeOffset } @@ -658,10 +625,10 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains query := ss.db. Model(&dbHost{}). Scopes( - hostFilter(filterMode, ss.hasAllowlist(), ss.hasBlocklist(), "hosts"), + hostFilter(filterMode, ss.hasAllowlist(), ss.hasBlocklist()), hostNetAddress(addressContains), hostPublicKey(keyIn), - ).Preload("Checks.DBAutopilot") + ) // preload allowlist and blocklist if filterMode == api.HostFilterModeAll { @@ -670,7 +637,10 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains Preload("Blocklist") } - var hosts []api.HostInfo + // preload host checks + query = query.Preload("Checks.DBAutopilot") + + var hosts []api.Host var fullHosts []dbHost err := query. Offset(offset). @@ -695,7 +665,7 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains } // Hosts returns non-blocked hosts at given offset and limit. -func (ss *SQLStore) Hosts(ctx context.Context, offset, limit int) ([]api.HostInfo, error) { +func (ss *SQLStore) Hosts(ctx context.Context, offset, limit int) ([]api.Host, error) { return ss.SearchHosts(ctx, api.HostFilterModeAllowed, "", nil, offset, limit) } @@ -1119,22 +1089,22 @@ func hostPublicKey(keyIn []types.PublicKey) func(*gorm.DB) *gorm.DB { // hostFilter can be used as a scope to filter hosts based on their filter mode, // returning either all, allowed or blocked hosts. -func hostFilter(filterMode string, hasAllowlist, hasBlocklist bool, hostTableAlias string) func(*gorm.DB) *gorm.DB { +func hostFilter(filterMode string, hasAllowlist, hasBlocklist bool) func(*gorm.DB) *gorm.DB { return func(db *gorm.DB) *gorm.DB { switch filterMode { case api.HostFilterModeAllowed: if hasAllowlist { - db = db.Where(fmt.Sprintf("EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", hostTableAlias)) + db = db.Where("EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") } if hasBlocklist { - db = db.Where(fmt.Sprintf("NOT EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", hostTableAlias)) + db = db.Where("NOT EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") } case api.HostFilterModeBlocked: if hasAllowlist { - db = db.Where(fmt.Sprintf("NOT EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", hostTableAlias)) + db = db.Where("NOT EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") } if hasBlocklist { - db = db.Where(fmt.Sprintf("EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = %s.id)", hostTableAlias)) + db = db.Where("EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") } if !hasAllowlist && !hasBlocklist { // if neither an allowlist nor a blocklist exist, all hosts are allowed diff --git a/worker/mocks_test.go b/worker/mocks_test.go index 6e324f67e..7e6d3afe9 100644 --- a/worker/mocks_test.go +++ b/worker/mocks_test.go @@ -260,13 +260,13 @@ var errSectorOutOfBounds = errors.New("sector out of bounds") type hostMock struct { hk types.PublicKey - hi api.HostInfo + hi api.Host } func newHostMock(hk types.PublicKey) *hostMock { return &hostMock{ hk: hk, - hi: api.HostInfo{ + hi: api.Host{ HostInfo: hostdb.HostInfo{ Host: hostdb.Host{ PublicKey: hk, @@ -289,13 +289,13 @@ func newHostStoreMock() *hostStoreMock { return &hostStoreMock{hosts: make(map[types.PublicKey]*hostMock)} } -func (hs *hostStoreMock) Host(ctx context.Context, hostKey types.PublicKey) (api.HostInfo, error) { +func (hs *hostStoreMock) Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) { hs.mu.Lock() defer hs.mu.Unlock() h, ok := hs.hosts[hostKey] if !ok { - return api.HostInfo{}, api.ErrHostNotFound + return api.Host{}, api.ErrHostNotFound } return h.hi, nil } diff --git a/worker/worker.go b/worker/worker.go index fda44dee6..7c14b1afd 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -110,7 +110,7 @@ type ( RecordPriceTables(ctx context.Context, priceTableUpdate []hostdb.PriceTableUpdate) error RecordContractSpending(ctx context.Context, records []api.ContractSpendingRecord) error - Host(ctx context.Context, hostKey types.PublicKey) (api.HostInfo, error) + Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) } ObjectStore interface { From 4d5c6491c7a8ee8b4caf6c38c62d5c7b90e14244 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 21 Mar 2024 20:51:48 +0100 Subject: [PATCH 06/16] api: cleanup api.Host type --- api/host.go | 5 ++-- autopilot/autopilot.go | 4 +-- autopilot/autopilot_test.go | 48 +++++++++++++++---------------- autopilot/contractor.go | 6 ++-- autopilot/hostfilter.go | 3 +- autopilot/hostinfo.go | 4 +-- autopilot/scanner_test.go | 10 ++----- bus/client/hosts.go | 2 +- hostdb/hostdb.go | 6 ---- internal/test/e2e/cluster_test.go | 4 +-- stores/hostdb.go | 48 +++++++++++++++---------------- worker/mocks_test.go | 8 ++---- 12 files changed, 67 insertions(+), 81 deletions(-) diff --git a/api/host.go b/api/host.go index 6c95165c9..cce3336ae 100644 --- a/api/host.go +++ b/api/host.go @@ -120,8 +120,9 @@ func (opts HostsForScanningOptions) Apply(values url.Values) { type ( Host struct { - hostdb.HostInfo - Checks map[string]HostCheck `json:"checks"` + hostdb.Host + Blocked bool `json:"blocked"` + Checks map[string]HostCheck `json:"checks"` } HostCheck struct { diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index b7b2f97d4..a9fa52343 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -740,7 +740,7 @@ func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.Host) (usables uint64) { gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow) for _, host := range hosts { - usable, _ := isUsableHost(cfg, rs, gc, host.HostInfo, smallestValidScore, 0) + usable, _ := isUsableHost(cfg, rs, gc, host, smallestValidScore, 0) if usable { usables++ } @@ -756,7 +756,7 @@ func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Cu resp.Hosts = uint64(len(hosts)) for _, host := range hosts { - usable, usableBreakdown := isUsableHost(cfg, rs, gc, host.HostInfo, 0, 0) + usable, usableBreakdown := isUsableHost(cfg, rs, gc, host, 0, 0) if usable { resp.Usable++ continue diff --git a/autopilot/autopilot_test.go b/autopilot/autopilot_test.go index 2edf88516..a21b55c7b 100644 --- a/autopilot/autopilot_test.go +++ b/autopilot/autopilot_test.go @@ -17,33 +17,33 @@ func TestOptimiseGougingSetting(t *testing.T) { var hosts []api.Host for i := 0; i < 10; i++ { hosts = append(hosts, api.Host{ - HostInfo: hostdb.HostInfo{ - Host: hostdb.Host{ - KnownSince: time.Unix(0, 0), - PriceTable: hostdb.HostPriceTable{ - HostPriceTable: rhpv3.HostPriceTable{ - CollateralCost: types.Siacoins(1), - MaxCollateral: types.Siacoins(1000), - }, - }, - Settings: rhpv2.HostSettings{ - AcceptingContracts: true, - Collateral: types.Siacoins(1), - MaxCollateral: types.Siacoins(1000), - Version: "1.6.0", - }, - Interactions: hostdb.Interactions{ - Uptime: time.Hour * 1000, - LastScan: time.Now(), - LastScanSuccess: true, - SecondToLastScanSuccess: true, - TotalScans: 100, + + Host: hostdb.Host{ + KnownSince: time.Unix(0, 0), + PriceTable: hostdb.HostPriceTable{ + HostPriceTable: rhpv3.HostPriceTable{ + CollateralCost: types.Siacoins(1), + MaxCollateral: types.Siacoins(1000), }, - LastAnnouncement: time.Unix(0, 0), - Scanned: true, }, - Blocked: false, + Settings: rhpv2.HostSettings{ + AcceptingContracts: true, + Collateral: types.Siacoins(1), + MaxCollateral: types.Siacoins(1000), + Version: "1.6.0", + }, + Interactions: hostdb.Interactions{ + Uptime: time.Hour * 1000, + LastScan: time.Now(), + LastScanSuccess: true, + SecondToLastScanSuccess: true, + TotalScans: 100, + }, + LastAnnouncement: time.Unix(0, 0), + Scanned: true, }, + Blocked: false, + Checks: nil, }) } diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 6f530d405..049059cad 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -295,7 +295,7 @@ func (c *contractor) performContractMaintenance(ctx context.Context, w Worker) ( for _, h := range hosts { // ignore the pricetable's HostBlockHeight by setting it to our own blockheight h.PriceTable.HostBlockHeight = cs.BlockHeight - isUsable, unusableResult := isUsableHost(state.cfg, state.rs, gc, h.HostInfo, minScore, hostData[h.PublicKey]) + isUsable, unusableResult := isUsableHost(state.cfg, state.rs, gc, h, minScore, hostData[h.PublicKey]) hostInfos[h.PublicKey] = hostInfo{ Usable: isUsable, UnusableResult: unusableResult, @@ -777,7 +777,7 @@ func (c *contractor) runContractChecks(ctx context.Context, w Worker, contracts host.PriceTable.HostBlockHeight = cs.BlockHeight // decide whether the host is still good - usable, unusableResult := isUsableHost(state.cfg, state.rs, gc, host.HostInfo, minScore, contract.FileSize()) + usable, unusableResult := isUsableHost(state.cfg, state.rs, gc, host, minScore, contract.FileSize()) if !usable { reasons := unusableResult.reasons() toStopUsing[fcid] = strings.Join(reasons, ",") @@ -1346,7 +1346,7 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []api.Host, usedH // NOTE: ignore the pricetable's HostBlockHeight by setting it to our // own blockheight h.PriceTable.HostBlockHeight = cs.BlockHeight - usable, result := isUsableHost(state.cfg, state.rs, gc, h.HostInfo, minScore, storedData[h.PublicKey]) + usable, result := isUsableHost(state.cfg, state.rs, gc, h, minScore, storedData[h.PublicKey]) if usable { candidates = append(candidates, scoredHost{h.Host, result.scoreBreakdown.Score()}) continue diff --git a/autopilot/hostfilter.go b/autopilot/hostfilter.go index 8de37221a..f41a20c94 100644 --- a/autopilot/hostfilter.go +++ b/autopilot/hostfilter.go @@ -11,7 +11,6 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/worker" ) @@ -176,7 +175,7 @@ func (u *unusableHostResult) keysAndValues() []interface{} { // isUsableHost returns whether the given host is usable along with a list of // reasons why it was deemed unusable. -func isUsableHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.GougingChecker, h hostdb.HostInfo, minScore float64, storedData uint64) (bool, unusableHostResult) { +func isUsableHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.GougingChecker, h api.Host, minScore float64, storedData uint64) (bool, unusableHostResult) { if rs.Validate() != nil { panic("invalid redundancy settings were supplied - developer error") } diff --git a/autopilot/hostinfo.go b/autopilot/hostinfo.go index d48ce7760..d82062a80 100644 --- a/autopilot/hostinfo.go +++ b/autopilot/hostinfo.go @@ -52,7 +52,7 @@ func (c *contractor) HostInfo(ctx context.Context, hostKey types.PublicKey) (api // ignore the pricetable's HostBlockHeight by setting it to our own blockheight host.Host.PriceTable.HostBlockHeight = cs.BlockHeight - isUsable, unusableResult := isUsableHost(state.cfg, rs, gc, host.HostInfo, minScore, storedData) + isUsable, unusableResult := isUsableHost(state.cfg, rs, gc, host, minScore, storedData) return api.HostHandlerResponse{ Host: host.Host, Checks: &api.HostHandlerResponseChecks{ @@ -89,7 +89,7 @@ func (c *contractor) hostInfoFromCache(ctx context.Context, host api.Host) (hi h } else { state := c.ap.State() gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) - isUsable, unusableResult := isUsableHost(state.cfg, state.rs, gc, host.HostInfo, minScore, storedData) + isUsable, unusableResult := isUsableHost(state.cfg, state.rs, gc, host, minScore, storedData) hi = hostInfo{ Usable: isUsable, UnusableResult: unusableResult, diff --git a/autopilot/scanner_test.go b/autopilot/scanner_test.go index 435e03b90..860a855fe 100644 --- a/autopilot/scanner_test.go +++ b/autopilot/scanner_test.go @@ -32,15 +32,11 @@ func (b *mockBus) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ( end = len(b.hosts) } - his := make([]api.Host, len(b.hosts[start:end])) + hosts := make([]api.Host, len(b.hosts[start:end])) for i, h := range b.hosts[start:end] { - his[i] = api.Host{ - HostInfo: hostdb.HostInfo{ - Host: h, - }, - } + hosts[i] = api.Host{Host: h} } - return his, nil + return hosts, nil } func (b *mockBus) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error) { diff --git a/bus/client/hosts.go b/bus/client/hosts.go index f0ab56fd1..8338d53f7 100644 --- a/bus/client/hosts.go +++ b/bus/client/hosts.go @@ -30,7 +30,7 @@ func (c *Client) HostBlocklist(ctx context.Context) (blocklist []string, err err } // Hosts returns 'limit' hosts at given 'offset'. -func (c *Client) Hosts(ctx context.Context, opts api.GetHostsOptions) (hosts []hostdb.HostInfo, err error) { +func (c *Client) Hosts(ctx context.Context, opts api.GetHostsOptions) (hosts []api.Host, err error) { values := url.Values{} opts.Apply(values) err = c.c.WithContext(ctx).GET("/hosts?"+values.Encode(), &hosts) diff --git a/hostdb/hostdb.go b/hostdb/hostdb.go index 69ed80989..1f4c341de 100644 --- a/hostdb/hostdb.go +++ b/hostdb/hostdb.go @@ -114,12 +114,6 @@ type HostPriceTable struct { Expiry time.Time `json:"expiry"` } -// HostInfo extends the host type with a field indicating whether it is blocked or not. -type HostInfo struct { - Host - Blocked bool `json:"blocked"` -} - // IsAnnounced returns whether the host has been announced. func (h Host) IsAnnounced() bool { return !h.LastAnnouncement.IsZero() diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index e081c2a5d..77898d4cf 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -166,7 +166,7 @@ func TestNewTestCluster(t *testing.T) { if len(hi.Checks.UnusableReasons) != 0 { t.Fatal("usable hosts don't have any reasons set") } - if reflect.DeepEqual(hi.Host, hostdb.HostInfo{}) { + if reflect.DeepEqual(hi.Host, hostdb.Host{}) { t.Fatal("host wasn't set") } } @@ -188,7 +188,7 @@ func TestNewTestCluster(t *testing.T) { if len(hi.Checks.UnusableReasons) != 0 { t.Fatal("usable hosts don't have any reasons set") } - if reflect.DeepEqual(hi.Host, hostdb.HostInfo{}) { + if reflect.DeepEqual(hi.Host, hostdb.Host{}) { t.Fatal("host wasn't set") } allHosts[hi.Host.PublicKey] = struct{}{} diff --git a/stores/hostdb.go b/stores/hostdb.go index f97af5fe9..b91f85601 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -322,33 +322,31 @@ func (h dbHost) convert(blocked bool) api.Host { checks[check.DBAutopilot.Identifier] = check.convert() } return api.Host{ - HostInfo: hostdb.HostInfo{ - Host: hostdb.Host{ - KnownSince: h.CreatedAt, - LastAnnouncement: h.LastAnnouncement, - NetAddress: h.NetAddress, - Interactions: hostdb.Interactions{ - TotalScans: h.TotalScans, - LastScan: lastScan, - LastScanSuccess: h.LastScanSuccess, - SecondToLastScanSuccess: h.SecondToLastScanSuccess, - Uptime: h.Uptime, - Downtime: h.Downtime, - SuccessfulInteractions: h.SuccessfulInteractions, - FailedInteractions: h.FailedInteractions, - LostSectors: h.LostSectors, - }, - PriceTable: hostdb.HostPriceTable{ - HostPriceTable: h.PriceTable.convert(), - Expiry: h.PriceTableExpiry.Time, - }, - PublicKey: types.PublicKey(h.PublicKey), - Scanned: h.Scanned, - Settings: h.Settings.convert(), + Host: hostdb.Host{ + KnownSince: h.CreatedAt, + LastAnnouncement: h.LastAnnouncement, + NetAddress: h.NetAddress, + Interactions: hostdb.Interactions{ + TotalScans: h.TotalScans, + LastScan: lastScan, + LastScanSuccess: h.LastScanSuccess, + SecondToLastScanSuccess: h.SecondToLastScanSuccess, + Uptime: h.Uptime, + Downtime: h.Downtime, + SuccessfulInteractions: h.SuccessfulInteractions, + FailedInteractions: h.FailedInteractions, + LostSectors: h.LostSectors, }, - Blocked: blocked, + PriceTable: hostdb.HostPriceTable{ + HostPriceTable: h.PriceTable.convert(), + Expiry: h.PriceTableExpiry.Time, + }, + PublicKey: types.PublicKey(h.PublicKey), + Scanned: h.Scanned, + Settings: h.Settings.convert(), }, - Checks: checks, + Blocked: blocked, + Checks: checks, } } diff --git a/worker/mocks_test.go b/worker/mocks_test.go index 7e6d3afe9..baf83b39d 100644 --- a/worker/mocks_test.go +++ b/worker/mocks_test.go @@ -267,11 +267,9 @@ func newHostMock(hk types.PublicKey) *hostMock { return &hostMock{ hk: hk, hi: api.Host{ - HostInfo: hostdb.HostInfo{ - Host: hostdb.Host{ - PublicKey: hk, - Scanned: true, - }, + Host: hostdb.Host{ + PublicKey: hk, + Scanned: true, }, }, } From 7675c8ed5ef8058da0875c38d420d6e9f69d7e17 Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 22 Mar 2024 09:38:55 +0100 Subject: [PATCH 07/16] api: revert SearchHostsRequest --- api/host.go | 7 +------ autopilot/autopilot.go | 2 +- autopilot/client.go | 16 +++++++--------- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/api/host.go b/api/host.go index cce3336ae..5221fb20c 100644 --- a/api/host.go +++ b/api/host.go @@ -43,18 +43,13 @@ type ( MinRecentScanFailures uint64 `json:"minRecentScanFailures"` } - // HostsRequest is the request type for the /api/autopilot/hosts endpoint. - HostsRequest struct { - UsabilityMode string `json:"usabilityMode"` - SearchHostsRequest - } - // SearchHostsRequest is the request type for the /api/bus/search/hosts // endpoint. SearchHostsRequest struct { Offset int `json:"offset"` Limit int `json:"limit"` FilterMode string `json:"filterMode"` + UsabilityMode string `json:"usabilityMode"` AddressContains string `json:"addressContains"` KeyIn []types.PublicKey `json:"keyIn"` } diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index a9fa52343..8127ade5f 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -726,7 +726,7 @@ func (ap *Autopilot) stateHandlerGET(jc jape.Context) { } func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { - var req api.HostsRequest + var req api.SearchHostsRequest if jc.Decode(&req) != nil { return } diff --git a/autopilot/client.go b/autopilot/client.go index 336149f8a..01d0a1632 100644 --- a/autopilot/client.go +++ b/autopilot/client.go @@ -41,15 +41,13 @@ func (c *Client) HostInfo(hostKey types.PublicKey) (resp api.HostHandlerResponse // HostInfo returns information about all hosts. func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostHandlerResponse, err error) { - err = c.c.POST("/hosts", api.HostsRequest{ - UsabilityMode: usabilityMode, - SearchHostsRequest: api.SearchHostsRequest{ - Offset: offset, - Limit: limit, - FilterMode: filterMode, - AddressContains: addressContains, - KeyIn: keyIn, - }, + err = c.c.POST("/hosts", api.SearchHostsRequest{ + Offset: offset, + Limit: limit, + FilterMode: filterMode, + UsabilityMode: usabilityMode, + AddressContains: addressContains, + KeyIn: keyIn, }, &resp) return } From ebf547dd337adf0ac64ee7f0bce38c0b43618967 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 02:05:55 +0000 Subject: [PATCH 08/16] build(deps): bump gorm.io/gorm from 1.25.7 to 1.25.8 Bumps [gorm.io/gorm](https://github.com/go-gorm/gorm) from 1.25.7 to 1.25.8. - [Release notes](https://github.com/go-gorm/gorm/releases) - [Commits](https://github.com/go-gorm/gorm/compare/v1.25.7...v1.25.8) --- updated-dependencies: - dependency-name: gorm.io/gorm dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0c813f7bb..ae14f6691 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/mysql v1.5.4 gorm.io/driver/sqlite v1.5.5 - gorm.io/gorm v1.25.7 + gorm.io/gorm v1.25.8 lukechampine.com/frand v1.4.2 moul.io/zapgorm2 v1.3.0 ) diff --git a/go.sum b/go.sum index d15acfcf9..f896c5c85 100644 --- a/go.sum +++ b/go.sum @@ -411,8 +411,8 @@ gorm.io/driver/sqlite v1.5.5 h1:7MDMtUZhV065SilG62E0MquljeArQZNfJnjd9i9gx3E= gorm.io/driver/sqlite v1.5.5/go.mod h1:6NgQ7sQWAIFsPrJJl1lSNSu2TABh0ZZ/zm5fosATavE= gorm.io/gorm v1.23.6/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= -gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A= -gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.8 h1:WAGEZ/aEcznN4D03laj8DKnehe1e9gYQAjW8xyPRdeo= +gorm.io/gorm v1.25.8/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw= lukechampine.com/frand v1.4.2/go.mod h1:4S/TM2ZgrKejMcKMbeLjISpJMO+/eZ1zu3vYX9dtj3s= From 7c47f98c8b4be2ffffff84b5829ac6fa06c793d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 02:06:11 +0000 Subject: [PATCH 09/16] build(deps): bump github.com/go-gormigrate/gormigrate/v2 Bumps [github.com/go-gormigrate/gormigrate/v2](https://github.com/go-gormigrate/gormigrate) from 2.1.1 to 2.1.2. - [Release notes](https://github.com/go-gormigrate/gormigrate/releases) - [Changelog](https://github.com/go-gormigrate/gormigrate/blob/master/CHANGELOG.md) - [Commits](https://github.com/go-gormigrate/gormigrate/compare/v2.1.1...v2.1.2) --- updated-dependencies: - dependency-name: github.com/go-gormigrate/gormigrate/v2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 0c813f7bb..696243c82 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21.6 require ( github.com/gabriel-vasile/mimetype v1.4.3 - github.com/go-gormigrate/gormigrate/v2 v2.1.1 + github.com/go-gormigrate/gormigrate/v2 v2.1.2 github.com/google/go-cmp v0.6.0 github.com/gotd/contrib v0.19.0 github.com/klauspost/reedsolomon v1.12.1 @@ -25,7 +25,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/mysql v1.5.4 gorm.io/driver/sqlite v1.5.5 - gorm.io/gorm v1.25.7 + gorm.io/gorm v1.25.8 lukechampine.com/frand v1.4.2 moul.io/zapgorm2 v1.3.0 ) diff --git a/go.sum b/go.sum index d15acfcf9..01dcd16ea 100644 --- a/go.sum +++ b/go.sum @@ -39,8 +39,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gormigrate/gormigrate/v2 v2.1.1 h1:eGS0WTFRV30r103lU8JNXY27KbviRnqqIDobW3EV3iY= -github.com/go-gormigrate/gormigrate/v2 v2.1.1/go.mod h1:L7nJ620PFDKei9QOhJzqA8kRCk+E3UbV2f5gv+1ndLc= +github.com/go-gormigrate/gormigrate/v2 v2.1.2 h1:F/d1hpHbRAvKezziV2CC5KUE82cVe9zTgHSBoOOZ4CY= +github.com/go-gormigrate/gormigrate/v2 v2.1.2/go.mod h1:9nHVX6z3FCMCQPA7PThGcA55t22yKQfK/Dnsf5i7hUo= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -411,8 +411,8 @@ gorm.io/driver/sqlite v1.5.5 h1:7MDMtUZhV065SilG62E0MquljeArQZNfJnjd9i9gx3E= gorm.io/driver/sqlite v1.5.5/go.mod h1:6NgQ7sQWAIFsPrJJl1lSNSu2TABh0ZZ/zm5fosATavE= gorm.io/gorm v1.23.6/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= -gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A= -gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.8 h1:WAGEZ/aEcznN4D03laj8DKnehe1e9gYQAjW8xyPRdeo= +gorm.io/gorm v1.25.8/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw= lukechampine.com/frand v1.4.2/go.mod h1:4S/TM2ZgrKejMcKMbeLjISpJMO+/eZ1zu3vYX9dtj3s= From 66f9df2c984eaa40866fc8a7a25253517658f100 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 25 Mar 2024 11:11:37 +0100 Subject: [PATCH 10/16] ci: make use of reusable project-add.yml --- .github/workflows/project-add.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/project-add.yml b/.github/workflows/project-add.yml index 3304fc0db..63df05bc2 100644 --- a/.github/workflows/project-add.yml +++ b/.github/workflows/project-add.yml @@ -10,12 +10,5 @@ on: jobs: add-to-project: - name: Add issue to project - runs-on: ubuntu-latest - steps: - - uses: actions/add-to-project@v0.5.0 - with: - # You can target a project in a different organization - # to the issue - project-url: https://github.com/orgs/SiaFoundation/projects/5 - github-token: ${{ secrets.PAT_ADD_TO_PROJECT }} + uses: SiaFoundation/workflows/.github/workflows/project-add.yml@master + From 11a4ff18bce249dbacfa654fbbef7db087215b15 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 25 Mar 2024 11:18:00 +0100 Subject: [PATCH 11/16] ci: inherit secrets --- .github/workflows/project-add.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/project-add.yml b/.github/workflows/project-add.yml index 63df05bc2..c61a17b0d 100644 --- a/.github/workflows/project-add.yml +++ b/.github/workflows/project-add.yml @@ -11,4 +11,5 @@ on: jobs: add-to-project: uses: SiaFoundation/workflows/.github/workflows/project-add.yml@master + secrets: inherit From 4b3968573d8fcaeda663f3ba119cb7313ecd76ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 10:31:31 +0000 Subject: [PATCH 12/16] build(deps): bump gorm.io/driver/mysql from 1.5.4 to 1.5.6 Bumps [gorm.io/driver/mysql](https://github.com/go-gorm/mysql) from 1.5.4 to 1.5.6. - [Commits](https://github.com/go-gorm/mysql/compare/v1.5.4...v1.5.6) --- updated-dependencies: - dependency-name: gorm.io/driver/mysql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 696243c82..22515806c 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( golang.org/x/crypto v0.21.0 golang.org/x/term v0.18.0 gopkg.in/yaml.v3 v3.0.1 - gorm.io/driver/mysql v1.5.4 + gorm.io/driver/mysql v1.5.6 gorm.io/driver/sqlite v1.5.5 gorm.io/gorm v1.25.8 lukechampine.com/frand v1.4.2 diff --git a/go.sum b/go.sum index 01dcd16ea..81612096a 100644 --- a/go.sum +++ b/go.sum @@ -405,12 +405,12 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/mysql v1.5.4 h1:igQmHfKcbaTVyAIHNhhB888vvxh8EdQ2uSUT0LPcBso= -gorm.io/driver/mysql v1.5.4/go.mod h1:9rYxJph/u9SWkWc9yY4XJ1F/+xO0S/ChOmbk3+Z5Tvs= +gorm.io/driver/mysql v1.5.6 h1:Ld4mkIickM+EliaQZQx3uOJDJHtrd70MxAUqWqlx3Y8= +gorm.io/driver/mysql v1.5.6/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= gorm.io/driver/sqlite v1.5.5 h1:7MDMtUZhV065SilG62E0MquljeArQZNfJnjd9i9gx3E= gorm.io/driver/sqlite v1.5.5/go.mod h1:6NgQ7sQWAIFsPrJJl1lSNSu2TABh0ZZ/zm5fosATavE= gorm.io/gorm v1.23.6/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.8 h1:WAGEZ/aEcznN4D03laj8DKnehe1e9gYQAjW8xyPRdeo= gorm.io/gorm v1.25.8/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 80eca756f0f280ae4c777a635266495d881ec35d Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 25 Mar 2024 15:46:53 +0100 Subject: [PATCH 13/16] worker: update logging in scanHost and apply timeout to each step of scanning --- worker/worker.go | 99 ++++++++++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 41 deletions(-) diff --git a/worker/worker.go b/worker/worker.go index d0de33f71..a39fc608e 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -1436,26 +1436,31 @@ func (w *worker) scanHost(ctx context.Context, timeout time.Duration, hostKey ty logger := w.logger.With("host", hostKey).With("hostIP", hostIP).With("timeout", timeout) // prepare a helper for scanning scan := func() (rhpv2.HostSettings, rhpv3.HostPriceTable, time.Duration, error) { - // apply timeout - scanCtx := ctx - var cancel context.CancelFunc - if timeout > 0 { - scanCtx, cancel = context.WithTimeout(scanCtx, timeout) - defer cancel() - } - // resolve hostIP. We don't want to scan hosts on private networks. - if !w.allowPrivateIPs { - host, _, err := net.SplitHostPort(hostIP) - if err != nil { - return rhpv2.HostSettings{}, rhpv3.HostPriceTable{}, 0, err + // helper to prepare a context for scanning + withTimeoutCtx := func() (context.Context, context.CancelFunc) { + if timeout > 0 { + return context.WithTimeout(ctx, timeout) } - addrs, err := (&net.Resolver{}).LookupIPAddr(scanCtx, host) - if err != nil { - return rhpv2.HostSettings{}, rhpv3.HostPriceTable{}, 0, err - } - for _, addr := range addrs { - if isPrivateIP(addr.IP) { - return rhpv2.HostSettings{}, rhpv3.HostPriceTable{}, 0, api.ErrHostOnPrivateNetwork + return ctx, func() {} + } + // resolve the address + { + scanCtx, cancel := withTimeoutCtx() + defer cancel() + // resolve hostIP. We don't want to scan hosts on private networks. + if !w.allowPrivateIPs { + host, _, err := net.SplitHostPort(hostIP) + if err != nil { + return rhpv2.HostSettings{}, rhpv3.HostPriceTable{}, 0, err + } + addrs, err := (&net.Resolver{}).LookupIPAddr(scanCtx, host) + if err != nil { + return rhpv2.HostSettings{}, rhpv3.HostPriceTable{}, 0, err + } + for _, addr := range addrs { + if isPrivateIP(addr.IP) { + return rhpv2.HostSettings{}, rhpv3.HostPriceTable{}, 0, api.ErrHostOnPrivateNetwork + } } } } @@ -1463,37 +1468,49 @@ func (w *worker) scanHost(ctx context.Context, timeout time.Duration, hostKey ty // fetch the host settings start := time.Now() var settings rhpv2.HostSettings - err := w.withTransportV2(scanCtx, hostKey, hostIP, func(t *rhpv2.Transport) error { - var err error - if settings, err = RPCSettings(scanCtx, t); err != nil { - return fmt.Errorf("failed to fetch host settings: %w", err) + { + scanCtx, cancel := withTimeoutCtx() + defer cancel() + err := w.withTransportV2(scanCtx, hostKey, hostIP, func(t *rhpv2.Transport) error { + var err error + if settings, err = RPCSettings(scanCtx, t); err != nil { + return fmt.Errorf("failed to fetch host settings: %w", err) + } + // NOTE: we overwrite the NetAddress with the host address here + // since we just used it to dial the host we know it's valid + settings.NetAddress = hostIP + return nil + }) + if err != nil { + return settings, rhpv3.HostPriceTable{}, time.Since(start), err } - // NOTE: we overwrite the NetAddress with the host address here - // since we just used it to dial the host we know it's valid - settings.NetAddress = hostIP - return nil - }) - elapsed := time.Since(start) - if err != nil { - return settings, rhpv3.HostPriceTable{}, elapsed, err } // fetch the host pricetable var pt rhpv3.HostPriceTable - err = w.transportPoolV3.withTransportV3(scanCtx, hostKey, settings.SiamuxAddr(), func(ctx context.Context, t *transportV3) error { - if hpt, err := RPCPriceTable(ctx, t, func(pt rhpv3.HostPriceTable) (rhpv3.PaymentMethod, error) { return nil, nil }); err != nil { - return fmt.Errorf("failed to fetch host price table: %w", err) - } else { - pt = hpt.HostPriceTable - return nil + { + scanCtx, cancel := withTimeoutCtx() + defer cancel() + err := w.transportPoolV3.withTransportV3(scanCtx, hostKey, settings.SiamuxAddr(), func(ctx context.Context, t *transportV3) error { + if hpt, err := RPCPriceTable(ctx, t, func(pt rhpv3.HostPriceTable) (rhpv3.PaymentMethod, error) { return nil, nil }); err != nil { + return fmt.Errorf("failed to fetch host price table: %w", err) + } else { + pt = hpt.HostPriceTable + return nil + } + }) + if err != nil { + return settings, rhpv3.HostPriceTable{}, time.Since(start), err } - }) - return settings, pt, elapsed, err + } + return settings, pt, time.Since(start), nil } // scan: first try settings, pt, duration, err := scan() if err != nil { + logger = logger.With(zap.Error(err)) + // scan: second try select { case <-ctx.Done(): @@ -1502,11 +1519,11 @@ func (w *worker) scanHost(ctx context.Context, timeout time.Duration, hostKey ty } settings, pt, duration, err = scan() - logger = logger.With("elapsed", duration) + logger = logger.With("elapsed", duration).With(zap.Error(err)) if err == nil { logger.Info("successfully scanned host on second try") } else if !isErrHostUnreachable(err) { - logger.Infow("failed to scan host", zap.Error(err)) + logger.Infow("failed to scan host") } } From b30999c07087e1fab4611915e93e63a650499ddd Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 25 Mar 2024 17:20:33 +0100 Subject: [PATCH 14/16] contractor: add forgiveness period for failed refreshes --- autopilot/contractor.go | 33 +++++++++++++++++++++++++++++++++ autopilot/contractor_test.go | 31 +++++++++++++++++++++++++++++++ autopilot/hostfilter.go | 4 ++-- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 7b2ea9863..82ea4e619 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -33,6 +33,10 @@ const ( // contract. estimatedFileContractTransactionSetSize = 2048 + // failedRenewalForgivenessPeriod is the amount of time we wait before + // punishing a contract for not being able to refresh + failedRefreshForgivenessPeriod = 24 * time.Hour + // leewayPctCandidateHosts is the leeway we apply when fetching candidate // hosts, we fetch ~10% more than required leewayPctCandidateHosts = 1.1 @@ -96,6 +100,8 @@ type ( revisionLastBroadcast map[types.FileContractID]time.Time revisionSubmissionBuffer uint64 + firstRefreshFailure map[types.FileContractID]time.Time + mu sync.Mutex pruning bool @@ -162,6 +168,8 @@ func newContractor(ap *Autopilot, revisionSubmissionBuffer uint64, revisionBroad revisionLastBroadcast: make(map[types.FileContractID]time.Time), revisionSubmissionBuffer: revisionSubmissionBuffer, + firstRefreshFailure: make(map[types.FileContractID]time.Time), + resolver: newIPResolver(ap.shutdownCtx, resolverLookupTimeout, ap.logger.Named("resolver")), } } @@ -226,6 +234,9 @@ func (c *contractor) performContractMaintenance(ctx context.Context, w Worker) ( contracts := resp.Contracts c.logger.Infof("fetched %d contracts from the worker, took %v", len(resp.Contracts), time.Since(start)) + // prune contract refresh failure map + c.pruneContractRefreshFailures(contracts) + // run revision broadcast c.runRevisionBroadcast(ctx, w, contracts, isInCurrentSet) @@ -1624,6 +1635,28 @@ func (c *contractor) hostForContract(ctx context.Context, fcid types.FileContrac return } +func (c *contractor) pruneContractRefreshFailures(contracts []api.Contract) { + contractMap := make(map[types.FileContractID]struct{}) + for _, contract := range contracts { + contractMap[contract.ID] = struct{}{} + } + for fcid := range c.firstRefreshFailure { + if _, ok := contractMap[fcid]; !ok { + delete(c.firstRefreshFailure, fcid) + } + } +} + +func (c *contractor) shouldForgiveFailedRefresh(fcid types.FileContractID) bool { + lastFailure, exists := c.firstRefreshFailure[fcid] + if !exists { + lastFailure = time.Now() + c.firstRefreshFailure[fcid] = lastFailure + } + fmt.Println(time.Since(lastFailure)) + return time.Since(lastFailure) < failedRefreshForgivenessPeriod +} + func addLeeway(n uint64, pct float64) uint64 { if pct < 0 { panic("given leeway percent has to be positive") diff --git a/autopilot/contractor_test.go b/autopilot/contractor_test.go index 575605612..9ce54daf5 100644 --- a/autopilot/contractor_test.go +++ b/autopilot/contractor_test.go @@ -3,8 +3,12 @@ package autopilot import ( "math" "testing" + "time" + "go.sia.tech/core/types" + "go.sia.tech/renterd/api" "go.uber.org/zap" + "lukechampine.com/frand" ) func TestCalculateMinScore(t *testing.T) { @@ -35,3 +39,30 @@ func TestCalculateMinScore(t *testing.T) { t.Fatalf("expected minScore to be math.SmallestNonzeroFLoat64 but was %v", minScore) } } + +func TestShouldForgiveFailedRenewal(t *testing.T) { + var fcid types.FileContractID + frand.Read(fcid[:]) + c := &contractor{ + firstRefreshFailure: make(map[types.FileContractID]time.Time), + } + + // try twice since the first time will set the failure time + if !c.shouldForgiveFailedRefresh(fcid) { + t.Fatal("should forgive") + } else if !c.shouldForgiveFailedRefresh(fcid) { + t.Fatal("should forgive") + } + + // set failure to be a full period in the past + c.firstRefreshFailure[fcid] = time.Now().Add(-failedRefreshForgivenessPeriod - time.Second) + if c.shouldForgiveFailedRefresh(fcid) { + t.Fatal("should not forgive") + } + + // prune map + c.pruneContractRefreshFailures([]api.Contract{}) + if len(c.firstRefreshFailure) != 0 { + t.Fatal("expected no failures") + } +} diff --git a/autopilot/hostfilter.go b/autopilot/hostfilter.go index f41a20c94..d64c1f3e3 100644 --- a/autopilot/hostfilter.go +++ b/autopilot/hostfilter.go @@ -254,8 +254,8 @@ func (c *contractor) isUsableContract(cfg api.AutopilotConfig, state state, ci c } if isOutOfFunds(cfg, pt, contract) { reasons = append(reasons, errContractOutOfFunds.Error()) - usable = false - recoverable = true + usable = usable && c.shouldForgiveFailedRefresh(contract.ID) + recoverable = !usable // only needs to be recoverable if !usable refresh = true renew = false } From 73f0640f2d5edaa0e043c95f57f37c9bf0278efd Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 25 Mar 2024 17:10:59 -0700 Subject: [PATCH 15/16] ci: release nightlies on linux --- .github/workflows/publish.yml | 56 ++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 824f69231..8b11e8b62 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,13 +1,13 @@ name: Publish -# Controls when the action will run. +# Controls when the action will run. on: # Triggers the workflow on new SemVer tags push: branches: - master - dev - tags: + tags: - 'v[0-9]+.[0-9]+.[0-9]+' - 'v[0-9]+.[0-9]+.[0-9]+-**' @@ -116,7 +116,7 @@ jobs: with: name: renterd path: release/ - build-mac: + build-mac: runs-on: macos-latest strategy: matrix: @@ -212,7 +212,7 @@ jobs: with: name: renterd path: release/ - build-windows: + build-windows: runs-on: windows-latest strategy: matrix: @@ -253,23 +253,21 @@ jobs: with: name: renterd path: release/ - dispatch: + + dispatch-homebrew: # only runs on full releases if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-') needs: [docker, build-linux, build-mac, build-windows] - strategy: - matrix: - repo: ['siafoundation/homebrew-sia', 'siafoundation/linux'] runs-on: ubuntu-latest steps: - name: Extract Tag Name id: get_tag run: echo "::set-output name=tag_name::${GITHUB_REF#refs/tags/}" - - name: Repository Dispatch + - name: Dispatch uses: peter-evans/repository-dispatch@v3 with: token: ${{ secrets.PAT_REPOSITORY_DISPATCH }} - repository: ${{ matrix.repo }} + repository: siafoundation/homebrew-sia event-type: release-tagged client-payload: > { @@ -277,4 +275,40 @@ jobs: "tag": "${{ steps.get_tag.outputs.tag_name }}", "project": "renterd", "workflow_id": "${{ github.run_id }}" - } \ No newline at end of file + } + dispatch-linux: # run on full releases, release candidates, and master branch + if: startsWith(github.ref, 'refs/tags/v') || endsWith(github.ref, 'master') + needs: [docker, build-linux, build-mac, build-windows] + runs-on: ubuntu-latest + steps: + - name: Build Dispatch Payload + id: get_payload + uses: actions/github-script@v7 + with: + script: | + const isRelease = context.ref.startsWith('refs/tags/v'), + isBeta = isRelease && context.ref.includes('-beta'), + tag = isRelease ? context.ref.replace('refs/tags/', '') : 'master'; + + let component = 'nightly'; + if (isBeta) { + component = 'beta'; + } else if (isRelease) { + component = 'main'; + } + + return { + description: "renterd: The Next-Gen Sia Renter", + tag: tag, + project: "renterd", + workflow_id: context.runId, + component: component + }; + + - name: Dispatch + uses: peter-evans/repository-dispatch@v3 + with: + token: ${{ secrets.PAT_REPOSITORY_DISPATCH }} + repository: siafoundation/linux + event-type: release-tagged + client-payload: ${{ steps.get_payload.outputs.result }} \ No newline at end of file From bcc4591e707b773eada22824c3dd16a30d04dd8f Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 27 Mar 2024 11:33:26 +0100 Subject: [PATCH 16/16] autopilot: remove debug logging --- autopilot/contractor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 82ea4e619..49ba304ae 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -1653,7 +1653,6 @@ func (c *contractor) shouldForgiveFailedRefresh(fcid types.FileContractID) bool lastFailure = time.Now() c.firstRefreshFailure[fcid] = lastFailure } - fmt.Println(time.Since(lastFailure)) return time.Since(lastFailure) < failedRefreshForgivenessPeriod }