Add basic metrics

api/client.go

@@ -115,3 +115,15 @@ func (c *Client) Contracts(ids []types.FileContractID) (resp []explorer.FileCont
 	err = c.c.POST("/explorer/contracts", ids, &resp)
 	return
 }
+
+// Metrics returns the most recent metrics about Sia.
+func (c *Client) Metrics() (resp explorer.Metrics, err error) {
+	err = c.c.GET("/explorer/metrics", &resp)
+	return
+}
+
+// MetricsID returns various metrics about Sia at the time of the given block ID.
+func (c *Client) MetricsID(id types.BlockID) (resp explorer.Metrics, err error) {
+	err = c.c.GET(fmt.Sprintf("/explorer/metrics/%s", id), &resp)
+	return
+}

api/server.go

@@ -44,6 +44,7 @@ type (
 		Tip() (types.ChainIndex, error)
 		Block(id types.BlockID) (explorer.Block, error)
 		BestTip(height uint64) (types.ChainIndex, error)
+		Metrics(id types.BlockID) (explorer.Metrics, error)
 		Transactions(ids []types.TransactionID) ([]explorer.Transaction, error)
 		Balance(address types.Address) (sc types.Currency, immatureSC types.Currency, sf uint64, err error)
 		UnspentSiacoinOutputs(address types.Address, offset, limit uint64) ([]explorer.SiacoinOutput, error)
@@ -152,6 +153,31 @@ func (s *server) explorerTipHeightHandler(jc jape.Context) {
 	jc.Encode(tip)
 }
 
+func (s *server) explorerMetricsHandler(jc jape.Context) {
+	tip, err := s.e.Tip()
+	if jc.Check("failed to get tip", err) != nil {
+		return
+	}
+
+	metrics, err := s.e.Metrics(tip.ID)
+	if jc.Check("failed to get metrics", err) != nil {
+		return
+	}
+	jc.Encode(metrics)
+}
+
+func (s *server) explorerMetricsIDHandler(jc jape.Context) {
+	var id types.BlockID
+	if jc.DecodeParam("id", &id) != nil {
+		return
+	}
+	metrics, err := s.e.Metrics(id)
+	if jc.Check("failed to get metrics", err) != nil {
+		return
+	}
+	jc.Encode(metrics)
+}
+
 func (s *server) explorerBlockHandler(jc jape.Context) {
 	var id types.BlockID
 	if jc.DecodeParam("id", &id) != nil {
@@ -313,6 +339,8 @@ func NewServer(e Explorer, cm ChainManager, s Syncer) http.Handler {
 		"GET    /explorer/tip":                        srv.explorerTipHandler,
 		"GET    /explorer/tip/:height":                srv.explorerTipHeightHandler,
 		"GET    /explorer/block/:id":                  srv.explorerBlockHandler,
+		"GET    /explorer/metrics":                    srv.explorerMetricsHandler,
+		"GET    /explorer/metrics/:id":                srv.explorerMetricsIDHandler,
 		"GET    /explorer/transactions/:id":           srv.explorerTransactionsIDHandler,
 		"POST   /explorer/transactions":               srv.explorerTransactionsHandler,
 		"GET    /explorer/addresses/:address/utxos":   srv.explorerAddressessAddressUtxosHandler,

explorer/events.go

@@ -16,6 +16,11 @@ const (
 	EventTypeFoundationSubsidy = "foundation subsidy"
 )
 
+// Arbitrary data specifiers
+var (
+	SpecifierAnnouncement = types.NewSpecifier("HostAnnouncement")
+)
+
 type eventData interface {
 	EventType() string
 }
@@ -186,9 +191,6 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate) []Event {
 	// handle v1 transactions
 	for _, txn := range b.Transactions {
 		relevant := relevantTxn(txn)
-		if len(relevant) == 0 {
-			continue
-		}
 
 		var e EventTransaction
 		for _, arb := range txn.ArbitraryData {
@@ -198,7 +200,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate) []Event {
 			prefix.DecodeFrom(d)
 			netAddress := d.ReadString()
 			uk.DecodeFrom(d)
-			if d.Err() == nil && prefix == types.NewSpecifier("HostAnnouncement") &&
+			if d.Err() == nil && prefix == SpecifierAnnouncement &&
 				uk.Algorithm == types.SpecifierEd25519 && len(uk.Key) == len(types.PublicKey{}) {
 				e.HostAnnouncements = append(e.HostAnnouncements, HostAnnouncement{
 					PublicKey:  *(*types.PublicKey)(uk.Key),
@@ -216,9 +218,6 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate) []Event {
 	// handle v2 transactions
 	for _, txn := range b.V2Transactions() {
 		relevant := relevantV2Txn(txn)
-		if len(relevant) == 0 {
-			continue
-		}
 
 		var e EventTransaction
 		for _, a := range txn.Attestations {

explorer/explorer.go

@@ -35,6 +35,7 @@ type Store interface {
 	Block(id types.BlockID) (Block, error)
 	BestTip(height uint64) (types.ChainIndex, error)
 	MerkleProof(leafIndex uint64) ([]types.Hash256, error)
+	Metrics(id types.BlockID) (Metrics, error)
 	Transactions(ids []types.TransactionID) ([]Transaction, error)
 	UnspentSiacoinOutputs(address types.Address, offset, limit uint64) ([]SiacoinOutput, error)
 	UnspentSiafundOutputs(address types.Address, offset, limit uint64) ([]SiafundOutput, error)
@@ -131,6 +132,11 @@ func (e *Explorer) MerkleProof(leafIndex uint64) ([]types.Hash256, error) {
 	return e.s.MerkleProof(leafIndex)
 }
 
+// Metrics returns various metrics about Sia.
+func (e *Explorer) Metrics(id types.BlockID) (Metrics, error) {
+	return e.s.Metrics(id)
+}
+
 // Transactions returns the transactions with the specified IDs.
 func (e *Explorer) Transactions(ids []types.TransactionID) ([]Transaction, error) {
 	return e.s.Transactions(ids)

explorer/types.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"time"
 
+	"go.sia.tech/core/consensus"
 	"go.sia.tech/core/types"
 )
 
@@ -104,7 +105,26 @@ type Block struct {
 
 	ParentID     types.BlockID   `json:"parentID"`
 	Nonce        uint64          `json:"nonce"`
+	Difficulty   consensus.Work  `json:"difficulty"`
 	Timestamp    time.Time       `json:"timestamp"`
 	MinerPayouts []SiacoinOutput `json:"minerPayouts"`
 	Transactions []Transaction   `json:"transactions"`
 }
+
+// Metrics contains various statistics relevant to the health of the Sia network.
+type Metrics struct {
+	// Current chain height
+	Height uint64 `json:"height"`
+	// Current difficulty
+	Difficulty consensus.Work `json:"difficulty"`
+	// Total announced hosts
+	TotalHosts uint64 `json:"totalHosts"`
+	// Number of active contracts
+	ActiveContracts uint64 `json:"activeContracts"`
+	// Number of failed contracts
+	FailedContracts uint64 `json:"failedContracts"`
+	// Number of successful contracts
+	SuccessfulContracts uint64 `json:"successfulContracts"`
+	// Current storage utilization, in bytes
+	StorageUtilization uint64 `json:"storageUtilization"`
+}

explorer/update.go

@@ -3,6 +3,7 @@ package explorer
 import (
 	"fmt"
 
+	"go.sia.tech/core/consensus"
 	"go.sia.tech/core/types"
 	"go.sia.tech/coreutils/chain"
 )
@@ -31,8 +32,9 @@ type (
 	// An UpdateState contains information relevant to the block being applied
 	// or reverted.
 	UpdateState struct {
-		Block types.Block
-		Index types.ChainIndex
+		Block      types.Block
+		Difficulty consensus.Work
+		Index      types.ChainIndex
 
 		Events      []Event
 		TreeUpdates []TreeNodeUpdate
@@ -156,8 +158,9 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error {
 
 	events := AppliedEvents(cau.State, cau.Block, cau)
 	state := UpdateState{
-		Block: cau.Block,
-		Index: cau.State.Index,
+		Block:      cau.Block,
+		Difficulty: cau.State.Difficulty,
+		Index:      cau.State.Index,
 
 		Events:      events,
 		TreeUpdates: treeUpdates,
@@ -252,6 +255,7 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types.
 
 	state := UpdateState{
 		Block:       cru.Block,
+		Difficulty:  cru.State.Difficulty,
 		Index:       revertedIndex,
 		TreeUpdates: treeUpdates,
 

persist/sqlite/blocks.go

@@ -10,7 +10,7 @@ import (
 // Block implements explorer.Store.
 func (s *Store) Block(id types.BlockID) (result explorer.Block, err error) {
 	err = s.transaction(func(tx *txn) error {
-		err = tx.QueryRow(`SELECT parent_id, nonce, timestamp, height FROM blocks WHERE id=?`, encode(id)).Scan(decode(&result.ParentID), decode(&result.Nonce), decode(&result.Timestamp), &result.Height)
+		err = tx.QueryRow(`SELECT parent_id, nonce, difficulty, timestamp, height FROM blocks WHERE id=?`, encode(id)).Scan(decode(&result.ParentID), decode(&result.Nonce), decode(&result.Difficulty), decode(&result.Timestamp), &result.Height)
 		if err != nil {
 			return err
 		}

persist/sqlite/consensus.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 
+	"go.sia.tech/core/consensus"
 	"go.sia.tech/core/types"
 	"go.sia.tech/coreutils/chain"
 	"go.sia.tech/explored/explorer"
@@ -16,9 +17,17 @@ type updateTx struct {
 	tx *txn
 }
 
-func addBlock(tx *txn, b types.Block, height uint64) error {
+func addBlock(tx *txn, b types.Block, height uint64, difficulty consensus.Work) error {
+	var totalHosts, activeContracts, failedContracts, successfulContracts, storageUtilization uint64
+	if height > 0 {
+		err := tx.QueryRow("SELECT total_hosts, active_contracts, failed_contracts, successful_contracts, storage_utilization from blocks WHERE height = ?", height-1).Scan(&totalHosts, &activeContracts, &failedContracts, &successfulContracts, &storageUtilization)
+		if err != nil {
+			return fmt.Errorf("addBlock: failed to get previous metrics: %w", err)
+		}
+	}
+
 	// nonce is encoded because database/sql doesn't support uint64 with high bit set
-	_, err := tx.Exec("INSERT INTO blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encode(b.Nonce), encode(b.Timestamp))
+	_, err := tx.Exec("INSERT INTO blocks(id, height, parent_id, nonce, timestamp, difficulty, total_hosts, active_contracts, failed_contracts, successful_contracts, storage_utilization) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encode(b.Nonce), encode(b.Timestamp), encode(difficulty), totalHosts, activeContracts, failedContracts, successfulContracts, storageUtilization)
 	return err
 }
 
@@ -754,6 +763,7 @@ func addFileContractElements(tx *txn, b types.Block, fces []explorer.FileContrac
 		if err != nil {
 			return fmt.Errorf("failed to execute file_contract_elements statement: %w", err)
 		}
+		// log.Printf("%v (%d) - resolved: %v, valid: %v", fcID, fc.RevisionNumber, resolved, valid)
 
 		// only update if it's the most recent revision which will come from
 		// running ForEachFileContractElement on the update
@@ -813,8 +823,69 @@ func addFileContractElements(tx *txn, b types.Block, fces []explorer.FileContrac
 	return fcDBIds, updateErr
 }
 
+func updateMetrics(tx *txn, b types.Block, fces []explorer.FileContractUpdate, events []explorer.Event) error {
+	hostExistsQuery, err := tx.Prepare(`SELECT NULL from host_announcements WHERE public_key = ?`)
+	if err != nil {
+		return fmt.Errorf("updateMetrics: failed to prepare host announcement query: %w", err)
+	}
+	updateQuery, err := tx.Prepare(`UPDATE blocks SET total_hosts = total_hosts + ?, active_contracts = active_contracts + ?, failed_contracts = failed_contracts + ?, successful_contracts = successful_contracts + ?, storage_utilization = storage_utilization + ? WHERE id = ?`)
+	if err != nil {
+		return fmt.Errorf("updateMetrics: failed to prepare metrics update query: %w", err)
+	}
+
+	var totalHostsDelta int64
+	for _, event := range events {
+		if event.Data.EventType() == explorer.EventTypeTransaction {
+			txn := event.Data.(*explorer.EventTransaction)
+			for _, host := range txn.HostAnnouncements {
+				// Technically, query.QueryRow().Err() should work here instead
+				// of doing an empty scan.  Unfortunately it appears to just
+				// returns nil, at least for the sqlite driver.
+				if err := hostExistsQuery.QueryRow(encode(host.PublicKey)).Scan(); errors.Is(err, sql.ErrNoRows) {
+					// we haven't seen this host yet
+					totalHostsDelta++
+				} else if err != nil {
+					return fmt.Errorf("updateMetrics: failed to find host announcement: %w", err)
+				}
+			}
+		}
+	}
+
+	var activeContractsDelta, failedContractsDelta, successfulContractsDelta, storageUtilizationDelta int64
+	for _, fce := range fces {
+		fc := fce.FileContractElement.FileContract
+		if fce.Revision != nil {
+			fc = fce.Revision.FileContract
+		}
+
+		if fce.Resolved {
+			activeContractsDelta--
+			storageUtilizationDelta -= int64(fc.Filesize)
+		} else if fce.Revision == nil {
+			// don't count revision as a new contract
+			activeContractsDelta++
+			storageUtilizationDelta += int64(fc.Filesize)
+		} else {
+			// filesize changed
+			storageUtilizationDelta += int64(fc.Filesize - fce.FileContractElement.FileContract.Filesize)
+		}
+
+		if !fce.Valid {
+			failedContractsDelta++
+		} else {
+			successfulContractsDelta++
+		}
+	}
+
+	_, err = updateQuery.Exec(totalHostsDelta, activeContractsDelta, failedContractsDelta, successfulContractsDelta, storageUtilizationDelta, encode(b.ID()))
+	if err != nil {
+		return fmt.Errorf("updateMetrics: failed to execute metrics update query: %w", err)
+	}
+	return nil
+}
+
 func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error {
-	if err := addBlock(ut.tx, state.Block, state.Index.Height); err != nil {
+	if err := addBlock(ut.tx, state.Block, state.Index.Height, state.Difficulty); err != nil {
 		return fmt.Errorf("ApplyIndex: failed to add block: %w", err)
 	} else if err := updateMaturedBalances(ut.tx, false, state.Index.Height); err != nil {
 		return fmt.Errorf("ApplyIndex: failed to update matured balances: %w", err)
@@ -853,6 +924,8 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error {
 		return fmt.Errorf("ApplyIndex: failed to add transactions: addTransactions: %w", err)
 	} else if err := updateStateTree(ut.tx, state.TreeUpdates); err != nil {
 		return fmt.Errorf("ApplyIndex: failed to update state tree: %w", err)
+	} else if err := updateMetrics(ut.tx, state.Block, state.FileContractElements, state.Events); err != nil {
+		return fmt.Errorf("ApplyIndex: failed to update metrics: %w", err)
 	} else if err := addEvents(ut.tx, scDBIds, fcDBIds, txnDBIds, state.Events); err != nil {
 		return fmt.Errorf("ApplyIndex: failed to add events: %w", err)
 	}