From 8eed8ff9fd8a349420c6f9410a13df69daa57d1b Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Thu, 12 Dec 2024 15:32:54 -0500 Subject: [PATCH] remove storage utilization, add total storage and remaining storage to host metrics --- api/api_test.go | 4 -- explorer/types.go | 15 ++++-- explorer/update.go | 3 -- persist/sqlite/consensus.go | 5 +- persist/sqlite/consensus_test.go | 88 ++++++++++++-------------------- persist/sqlite/init.sql | 1 - persist/sqlite/metrics.go | 5 +- 7 files changed, 50 insertions(+), 71 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index e46273d..e9150a7 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -111,8 +111,6 @@ func TestAPI(t *testing.T) { hostPrivateKey := types.GeneratePrivateKey() hostPublicKey := hostPrivateKey.PublicKey() - contractFilesize := uint64(10) - network, genesisBlock := ctestutil.Network() genesisBlock.Transactions[0].SiacoinOutputs[0].Address = addr1 genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr1 @@ -297,7 +295,6 @@ func TestAPI(t *testing.T) { testutil.Equal(t, "active contracts", 1, resp.ActiveContracts) testutil.Equal(t, "failed contracts", 0, resp.FailedContracts) testutil.Equal(t, "failed contracts", 0, resp.SuccessfulContracts) - testutil.Equal(t, "storage utilization", contractFilesize, resp.StorageUtilization) testutil.Equal(t, "contract revenue", types.ZeroCurrency, resp.ContractRevenue) }}, {"BlockMetricsID", func(t *testing.T) { @@ -318,7 +315,6 @@ func TestAPI(t *testing.T) { testutil.Equal(t, "active contracts", 1, resp.ActiveContracts) testutil.Equal(t, "failed contracts", 0, resp.FailedContracts) testutil.Equal(t, "failed contracts", 0, resp.SuccessfulContracts) - testutil.Equal(t, "storage utilization", contractFilesize, resp.StorageUtilization) testutil.Equal(t, "contract revenue", types.ZeroCurrency, resp.ContractRevenue) }}, {"Block", func(t *testing.T) { diff --git a/explorer/types.go b/explorer/types.go index eec17fa..23b99eb 100644 --- a/explorer/types.go +++ b/explorer/types.go @@ -295,8 +295,6 @@ type Metrics struct { FailedContracts uint64 `json:"failedContracts"` // Number of successful contracts SuccessfulContracts uint64 `json:"successfulContracts"` - // Current storage utilization, in bytes - StorageUtilization uint64 `json:"storageUtilization"` // Current circulating supply CirculatingSupply types.Currency `json:"circulatingSupply"` // Total contract revenue @@ -357,7 +355,14 @@ func (h Host) IsV2() bool { // HostMetrics represents averages of scanned information from hosts. type HostMetrics struct { - ActiveHosts uint64 `json:"activeHosts"` - Settings rhpv2.HostSettings `json:"settings"` - PriceTable rhpv3.HostPriceTable `json:"priceTable"` + // Number of hosts that were up as of there last scan + ActiveHosts uint64 `json:"activeHosts"` + // Total storage of all active hosts, in bytes + TotalStorage uint64 `json:"totalStorage"` + // Remaining storage of all active hosts, in bytes (storage utilization is + // equal to TotalStorage - RemainingStorage) + RemainingStorage uint64 `json:"remainingStorage"` + + Settings rhpv2.HostSettings `json:"settings"` + PriceTable rhpv3.HostPriceTable `json:"priceTable"` } diff --git a/explorer/update.go b/explorer/update.go index 6c0a9e8..1dc7c05 100644 --- a/explorer/update.go +++ b/explorer/update.go @@ -458,14 +458,11 @@ func updateMetrics(tx UpdateTx, s UpdateState, metrics Metrics) (Metrics, error) if fce.Resolved { metrics.ActiveContracts-- - metrics.StorageUtilization -= fc.Filesize } else if fce.Revision == nil { // don't count revision as a new contract metrics.ActiveContracts++ - metrics.StorageUtilization += fc.Filesize } else { // filesize changed - metrics.StorageUtilization += (fc.Filesize - fce.FileContractElement.FileContract.Filesize) } if fce.Resolved { diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 8330cbd..787a331 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -973,7 +973,7 @@ func updateFileContractIndices(tx *txn, revert bool, index types.ChainIndex, fce } func addMetrics(tx *txn, s explorer.UpdateState) error { - _, err := tx.Exec(`INSERT INTO network_metrics(block_id, height, difficulty, siafund_tax_revenue, num_leaves, total_hosts, active_contracts, failed_contracts, successful_contracts, storage_utilization, circulating_supply, contract_revenue) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + _, err := tx.Exec(`INSERT INTO network_metrics(block_id, height, difficulty, siafund_tax_revenue, num_leaves, total_hosts, active_contracts, failed_contracts, successful_contracts, circulating_supply, contract_revenue) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, encode(s.Metrics.Index.ID), s.Metrics.Index.Height, encode(s.Metrics.Difficulty), @@ -983,7 +983,6 @@ func addMetrics(tx *txn, s explorer.UpdateState) error { s.Metrics.ActiveContracts, s.Metrics.FailedContracts, s.Metrics.SuccessfulContracts, - s.Metrics.StorageUtilization, encode(s.Metrics.CirculatingSupply), encode(s.Metrics.ContractRevenue), ) @@ -997,7 +996,7 @@ func (ut *updateTx) HostExists(pubkey types.PublicKey) (exists bool, err error) func (ut *updateTx) Metrics(height uint64) (explorer.Metrics, error) { var metrics explorer.Metrics - if err := ut.tx.QueryRow("SELECT total_hosts, active_contracts, failed_contracts, successful_contracts, storage_utilization, circulating_supply, contract_revenue from network_metrics WHERE height = ?", height).Scan(&metrics.TotalHosts, &metrics.ActiveContracts, &metrics.FailedContracts, &metrics.SuccessfulContracts, &metrics.StorageUtilization, decode(&metrics.CirculatingSupply), decode(&metrics.ContractRevenue)); err != nil && err != sql.ErrNoRows { + if err := ut.tx.QueryRow("SELECT total_hosts, active_contracts, failed_contracts, successful_contracts, circulating_supply, contract_revenue from network_metrics WHERE height = ?", height).Scan(&metrics.TotalHosts, &metrics.ActiveContracts, &metrics.FailedContracts, &metrics.SuccessfulContracts, decode(&metrics.CirculatingSupply), decode(&metrics.ContractRevenue)); err != nil && err != sql.ErrNoRows { return explorer.Metrics{}, err } return metrics, nil diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index f5dffb7..65f8b35 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -103,7 +103,6 @@ func CheckMetrics(t *testing.T, db explorer.Store, cm *chain.Manager, expected e testutil.Equal(t, "failed contracts", expected.FailedContracts, got.FailedContracts) testutil.Equal(t, "successful contracts", expected.SuccessfulContracts, got.SuccessfulContracts) testutil.Equal(t, "contract revenue", expected.ContractRevenue, got.ContractRevenue) - testutil.Equal(t, "storage utilization", expected.StorageUtilization, got.StorageUtilization) // don't check circulating supply here because it requires a lot of accounting } @@ -617,9 +616,8 @@ func TestFileContract(t *testing.T) { } CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 0, - ActiveContracts: 1, - StorageUtilization: testutil.ContractFilesize, + TotalHosts: 0, + ActiveContracts: 1, }) // Explorer.Contracts should return latest revision @@ -660,9 +658,8 @@ func TestFileContract(t *testing.T) { for i := cm.Tip().Height; i < windowEnd; i++ { CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 0, - ActiveContracts: 1, - StorageUtilization: 1 * testutil.ContractFilesize, + TotalHosts: 0, + ActiveContracts: 1, }) if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { @@ -676,7 +673,6 @@ func TestFileContract(t *testing.T) { ActiveContracts: 0, FailedContracts: 1, SuccessfulContracts: 0, - StorageUtilization: 0, }) { @@ -725,7 +721,6 @@ func TestFileContract(t *testing.T) { ActiveContracts: 0, FailedContracts: 1, SuccessfulContracts: 0, - StorageUtilization: 0, }) } @@ -792,9 +787,8 @@ func TestEphemeralFileContract(t *testing.T) { confirmationTransactionID := txn.ID() CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 0, - ActiveContracts: 1, - StorageUtilization: testutil.ContractFilesize, + TotalHosts: 0, + ActiveContracts: 1, }) { @@ -889,9 +883,8 @@ func TestEphemeralFileContract(t *testing.T) { syncDB(t, db, cm) CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 0, - ActiveContracts: 1, - StorageUtilization: testutil.ContractFilesize, + TotalHosts: 0, + ActiveContracts: 1, }) // Explorer.Contracts should return latest revision @@ -982,9 +975,8 @@ func TestRevertTip(t *testing.T) { } CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 0, - ActiveContracts: 0, - StorageUtilization: 0, + TotalHosts: 0, + ActiveContracts: 0, }) { @@ -1009,9 +1001,8 @@ func TestRevertTip(t *testing.T) { } CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 0, - ActiveContracts: 0, - StorageUtilization: 0, + TotalHosts: 0, + ActiveContracts: 0, }) for i := 0; i < n; i++ { @@ -1087,9 +1078,8 @@ func TestRevertBalance(t *testing.T) { syncDB(t, db, cm) CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 0, - ActiveContracts: 0, - StorageUtilization: 0, + TotalHosts: 0, + ActiveContracts: 0, }) } testutil.CheckBalance(t, db, addr1, types.ZeroCurrency, types.ZeroCurrency, 0) @@ -1159,9 +1149,8 @@ func TestRevertBalance(t *testing.T) { } CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 0, - ActiveContracts: 0, - StorageUtilization: 0, + TotalHosts: 0, + ActiveContracts: 0, }) testutil.CheckBalance(t, db, addr1, hundredSC, types.ZeroCurrency, 0) @@ -1347,9 +1336,8 @@ func TestRevertSendTransactions(t *testing.T) { syncDB(t, db, cm) CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 0, - ActiveContracts: 0, - StorageUtilization: 0, + TotalHosts: 0, + ActiveContracts: 0, }) testutil.CheckBalance(t, db, addr1, addr1SCs, types.ZeroCurrency, addr1SFs) @@ -1518,9 +1506,8 @@ func TestRevertSendTransactions(t *testing.T) { } CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 0, - ActiveContracts: 0, - StorageUtilization: 0, + TotalHosts: 0, + ActiveContracts: 0, }) } @@ -1558,9 +1545,8 @@ func TestHostAnnouncement(t *testing.T) { syncDB(t, db, cm) CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 1, - ActiveContracts: 0, - StorageUtilization: 0, + TotalHosts: 1, + ActiveContracts: 0, }) txn2 := types.Transaction{ @@ -1586,9 +1572,8 @@ func TestHostAnnouncement(t *testing.T) { syncDB(t, db, cm) CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 3, - ActiveContracts: 0, - StorageUtilization: 0, + TotalHosts: 3, + ActiveContracts: 0, }) { @@ -1764,9 +1749,8 @@ func TestMultipleReorg(t *testing.T) { syncDB(t, db, cm) CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 0, - ActiveContracts: 0, - StorageUtilization: 0, + TotalHosts: 0, + ActiveContracts: 0, }) { @@ -2072,9 +2056,8 @@ func TestMultipleReorgFileContract(t *testing.T) { confirmationTransactionID := txn.ID() CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 0, - ActiveContracts: 1, - StorageUtilization: testutil.ContractFilesize, + TotalHosts: 0, + ActiveContracts: 1, }) { @@ -2136,9 +2119,8 @@ func TestMultipleReorgFileContract(t *testing.T) { prevState2 := cm.TipState() CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 0, - ActiveContracts: 1, - StorageUtilization: testutil.ContractFilesize + 10, + TotalHosts: 0, + ActiveContracts: 1, }) // Explorer.Contracts should return latest revision @@ -2240,9 +2222,8 @@ func TestMultipleReorgFileContract(t *testing.T) { // storage utilization should be back to testutil.ContractFilesize instead of // testutil.ContractFilesize + 10 CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 0, - ActiveContracts: 1, - StorageUtilization: testutil.ContractFilesize, + TotalHosts: 0, + ActiveContracts: 1, }) } @@ -2282,9 +2263,8 @@ func TestMultipleReorgFileContract(t *testing.T) { // should have revision filesize CheckMetrics(t, db, cm, explorer.Metrics{ - TotalHosts: 0, - ActiveContracts: 1, - StorageUtilization: testutil.ContractFilesize + 10, + TotalHosts: 0, + ActiveContracts: 1, }) } diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index f9675b6..cf0eeea 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -27,7 +27,6 @@ CREATE TABLE network_metrics ( active_contracts INTEGER NOT NULL, failed_contracts INTEGER NOT NULL, successful_contracts INTEGER NOT NULL, - storage_utilization INTEGER NOT NULL, circulating_supply BLOB NOT NULL, contract_revenue BLOB NOT NULL ); diff --git a/persist/sqlite/metrics.go b/persist/sqlite/metrics.go index 48074f6..a5f953a 100644 --- a/persist/sqlite/metrics.go +++ b/persist/sqlite/metrics.go @@ -11,7 +11,7 @@ import ( // Metrics implements explorer.Store func (s *Store) Metrics(id types.BlockID) (result explorer.Metrics, err error) { err = s.transaction(func(tx *txn) error { - err = tx.QueryRow(`SELECT block_id, height, difficulty, siafund_tax_revenue, num_leaves, total_hosts, active_contracts, failed_contracts, successful_contracts, storage_utilization, circulating_supply, contract_revenue FROM network_metrics WHERE block_id = ?`, encode(id)).Scan(decode(&result.Index.ID), &result.Index.Height, decode(&result.Difficulty), decode(&result.SiafundTaxRevenue), decode(&result.NumLeaves), &result.TotalHosts, &result.ActiveContracts, &result.FailedContracts, &result.SuccessfulContracts, &result.StorageUtilization, decode(&result.CirculatingSupply), decode(&result.ContractRevenue)) + err = tx.QueryRow(`SELECT block_id, height, difficulty, siafund_tax_revenue, num_leaves, total_hosts, active_contracts, failed_contracts, successful_contracts, circulating_supply, contract_revenue FROM network_metrics WHERE block_id = ?`, encode(id)).Scan(decode(&result.Index.ID), &result.Index.Height, decode(&result.Difficulty), decode(&result.SiafundTaxRevenue), decode(&result.NumLeaves), &result.TotalHosts, &result.ActiveContracts, &result.FailedContracts, &result.SuccessfulContracts, decode(&result.CirculatingSupply), decode(&result.ContractRevenue)) if err != nil { return fmt.Errorf("failed to get metrics: %w", err) } @@ -36,6 +36,9 @@ func (s *Store) HostMetrics() (result explorer.HostMetrics, err error) { return fmt.Errorf("failed to scan host: %w", err) } + result.TotalStorage += host.Settings.TotalStorage + result.RemainingStorage += host.Settings.RemainingStorage + result.Settings.MaxDownloadBatchSize += host.Settings.MaxDownloadBatchSize result.Settings.MaxDuration += host.Settings.MaxDuration result.Settings.MaxReviseBatchSize += host.Settings.MaxReviseBatchSize