Add basic metrics

api/client.go

@@ -115,3 +115,15 @@ func (c *Client) Contracts(ids []types.FileContractID) (resp []explorer.FileCont
 	err = c.c.POST("/explorer/contracts", ids, &resp)
 	return
 }
+
+// Metrics returns the most recent metrics about Sia.
+func (c *Client) Metrics() (resp explorer.Metrics, err error) {
+	err = c.c.GET("/explorer/metrics", &resp)
+	return
+}
+
+// MetricsID returns various metrics about Sia at the time of the given block ID.
+func (c *Client) MetricsID(id types.BlockID) (resp explorer.Metrics, err error) {
+	err = c.c.GET(fmt.Sprintf("/explorer/metrics/%s", id), &resp)
+	return
+}

api/server.go

@@ -44,6 +44,7 @@ type (
 		Tip() (types.ChainIndex, error)
 		Block(id types.BlockID) (explorer.Block, error)
 		BestTip(height uint64) (types.ChainIndex, error)
+		Metrics(id types.BlockID) (explorer.Metrics, error)
 		Transactions(ids []types.TransactionID) ([]explorer.Transaction, error)
 		Balance(address types.Address) (sc types.Currency, immatureSC types.Currency, sf uint64, err error)
 		UnspentSiacoinOutputs(address types.Address, offset, limit uint64) ([]explorer.SiacoinOutput, error)
@@ -152,6 +153,31 @@ func (s *server) explorerTipHeightHandler(jc jape.Context) {
 	jc.Encode(tip)
 }
 
+func (s *server) explorerMetricsHandler(jc jape.Context) {
+	tip, err := s.e.Tip()
+	if jc.Check("failed to get tip", err) != nil {
+		return
+	}
+
+	metrics, err := s.e.Metrics(tip.ID)
+	if jc.Check("failed to get metrics", err) != nil {
+		return
+	}
+	jc.Encode(metrics)
+}
+
+func (s *server) explorerMetricsIDHandler(jc jape.Context) {
+	var id types.BlockID
+	if jc.DecodeParam("id", &id) != nil {
+		return
+	}
+	metrics, err := s.e.Metrics(id)
+	if jc.Check("failed to get metrics", err) != nil {
+		return
+	}
+	jc.Encode(metrics)
+}
+
 func (s *server) explorerBlockHandler(jc jape.Context) {
 	var id types.BlockID
 	if jc.DecodeParam("id", &id) != nil {
@@ -313,6 +339,8 @@ func NewServer(e Explorer, cm ChainManager, s Syncer) http.Handler {
 		"GET    /explorer/tip":                        srv.explorerTipHandler,
 		"GET    /explorer/tip/:height":                srv.explorerTipHeightHandler,
 		"GET    /explorer/block/:id":                  srv.explorerBlockHandler,
+		"GET    /explorer/metrics":                    srv.explorerMetricsHandler,
+		"GET    /explorer/metrics/:id":                srv.explorerMetricsIDHandler,
 		"GET    /explorer/transactions/:id":           srv.explorerTransactionsIDHandler,
 		"POST   /explorer/transactions":               srv.explorerTransactionsHandler,
 		"GET    /explorer/addresses/:address/utxos":   srv.explorerAddressessAddressUtxosHandler,

explorer/events.go

@@ -16,6 +16,11 @@ const (
 	EventTypeFoundationSubsidy = "foundation subsidy"
 )
 
+// Arbitrary data specifiers
+var (
+	SpecifierAnnouncement = types.NewSpecifier("HostAnnouncement")
+)
+
 type eventData interface {
 	EventType() string
 }
@@ -186,9 +191,6 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate) []Event {
 	// handle v1 transactions
 	for _, txn := range b.Transactions {
 		relevant := relevantTxn(txn)
-		if len(relevant) == 0 {
-			continue
-		}
 
 		var e EventTransaction
 		for _, arb := range txn.ArbitraryData {
@@ -198,7 +200,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate) []Event {
 			prefix.DecodeFrom(d)
 			netAddress := d.ReadString()
 			uk.DecodeFrom(d)
-			if d.Err() == nil && prefix == types.NewSpecifier("HostAnnouncement") &&
+			if d.Err() == nil && prefix == SpecifierAnnouncement &&
 				uk.Algorithm == types.SpecifierEd25519 && len(uk.Key) == len(types.PublicKey{}) {
 				e.HostAnnouncements = append(e.HostAnnouncements, HostAnnouncement{
 					PublicKey:  *(*types.PublicKey)(uk.Key),
@@ -216,9 +218,6 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate) []Event {
 	// handle v2 transactions
 	for _, txn := range b.V2Transactions() {
 		relevant := relevantV2Txn(txn)
-		if len(relevant) == 0 {
-			continue
-		}
 
 		var e EventTransaction
 		for _, a := range txn.Attestations {

explorer/explorer.go

@@ -35,6 +35,7 @@ type Store interface {
 	Block(id types.BlockID) (Block, error)
 	BestTip(height uint64) (types.ChainIndex, error)
 	MerkleProof(leafIndex uint64) ([]types.Hash256, error)
+	Metrics(id types.BlockID) (Metrics, error)
 	Transactions(ids []types.TransactionID) ([]Transaction, error)
 	UnspentSiacoinOutputs(address types.Address, offset, limit uint64) ([]SiacoinOutput, error)
 	UnspentSiafundOutputs(address types.Address, offset, limit uint64) ([]SiafundOutput, error)
@@ -131,6 +132,11 @@ func (e *Explorer) MerkleProof(leafIndex uint64) ([]types.Hash256, error) {
 	return e.s.MerkleProof(leafIndex)
 }
 
+// Metrics returns various metrics about Sia.
+func (e *Explorer) Metrics(id types.BlockID) (Metrics, error) {
+	return e.s.Metrics(id)
+}
+
 // Transactions returns the transactions with the specified IDs.
 func (e *Explorer) Transactions(ids []types.TransactionID) ([]Transaction, error) {
 	return e.s.Transactions(ids)

explorer/types.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"time"
 
+	"go.sia.tech/core/consensus"
 	"go.sia.tech/core/types"
 )
 
@@ -108,3 +109,25 @@ type Block struct {
 	MinerPayouts []SiacoinOutput `json:"minerPayouts"`
 	Transactions []Transaction   `json:"transactions"`
 }
+
+// Metrics contains various statistics relevant to the health of the Sia network.
+type Metrics struct {
+	// Current chain height
+	Height uint64 `json:"height"`
+	// Current difficulty
+	Difficulty consensus.Work `json:"difficulty"`
+	// Total announced hosts
+	TotalHosts uint64 `json:"totalHosts"`
+	// Number of active contracts
+	ActiveContracts uint64 `json:"activeContracts"`
+	// Number of failed contracts
+	FailedContracts uint64 `json:"failedContracts"`
+	// Number of successful contracts
+	SuccessfulContracts uint64 `json:"successfulContracts"`
+	// Current storage utilization, in bytes
+	StorageUtilization uint64 `json:"storageUtilization"`
+	// Current circulating supply
+	CirculatingSupply types.Currency `json:"circulatingSupply"`
+	// Total contract revenue
+	ContractRevenue types.Currency `json:"contractRevenue"`
+}

explorer/update.go

@@ -3,6 +3,7 @@ package explorer
 import (
 	"fmt"
 
+	"go.sia.tech/core/consensus"
 	"go.sia.tech/core/types"
 	"go.sia.tech/coreutils/chain"
 )
@@ -31,8 +32,9 @@ type (
 	// An UpdateState contains information relevant to the block being applied
 	// or reverted.
 	UpdateState struct {
-		Block types.Block
-		Index types.ChainIndex
+		Block      types.Block
+		Difficulty consensus.Work
+		Index      types.ChainIndex
 
 		Events      []Event
 		TreeUpdates []TreeNodeUpdate
@@ -156,8 +158,9 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error {
 
 	events := AppliedEvents(cau.State, cau.Block, cau)
 	state := UpdateState{
-		Block: cau.Block,
-		Index: cau.State.Index,
+		Block:      cau.Block,
+		Difficulty: cau.State.Difficulty,
+		Index:      cau.State.Index,
 
 		Events:      events,
 		TreeUpdates: treeUpdates,
@@ -252,6 +255,7 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types.
 
 	state := UpdateState{
 		Block:       cru.Block,
+		Difficulty:  cru.State.Difficulty,
 		Index:       revertedIndex,
 		TreeUpdates: treeUpdates,
 

persist/sqlite/consensus.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 
+	"go.sia.tech/core/consensus"
 	"go.sia.tech/core/types"
 	"go.sia.tech/coreutils/chain"
 	"go.sia.tech/explored/explorer"
@@ -813,6 +814,99 @@ func addFileContractElements(tx *txn, b types.Block, fces []explorer.FileContrac
 	return fcDBIds, updateErr
 }
 
+func addMetrics(tx *txn, height uint64, difficulty consensus.Work, b types.Block, fces []explorer.FileContractUpdate, events []explorer.Event) error {
+	var existingMetrics explorer.Metrics
+	if height > 0 {
+		err := tx.QueryRow("SELECT total_hosts, active_contracts, failed_contracts, successful_contracts, storage_utilization, circulating_supply, contract_revenue from network_metrics WHERE height = ?", height-1).Scan(&existingMetrics.TotalHosts, &existingMetrics.ActiveContracts, &existingMetrics.FailedContracts, &existingMetrics.SuccessfulContracts, &existingMetrics.StorageUtilization, decode(&existingMetrics.CirculatingSupply), decode(&existingMetrics.ContractRevenue))
+		if err != nil {
+			return fmt.Errorf("addMetrics: failed to get previous metrics: %w", err)
+		}
+	} else {
+		// add genesis outputs
+		for _, txn := range b.Transactions {
+			for _, sco := range txn.SiacoinOutputs {
+				existingMetrics.CirculatingSupply = existingMetrics.CirculatingSupply.Add(sco.Value)
+			}
+		}
+	}
+
+	hostExistsQuery, err := tx.Prepare(`SELECT EXISTS(SELECT public_key FROM host_announcements WHERE public_key = ?)`)
+	if err != nil {
+		return fmt.Errorf("addMetrics: failed to prepare host announcement query: %w", err)
+	}
+	defer hostExistsQuery.Close()
+
+	var totalHostsDelta int64
+	for _, event := range events {
+		if event.Data.EventType() == explorer.EventTypeTransaction {
+			txn := event.Data.(*explorer.EventTransaction)
+			for _, host := range txn.HostAnnouncements {
+				var exists bool
+				if err := hostExistsQuery.QueryRow(encode(host.PublicKey)).Scan(&exists); err != nil {
+					return fmt.Errorf("failed to check host announcement: %w", err)
+				} else if !exists {
+					// we haven't seen this host yet
+					totalHostsDelta++
+				}
+			}
+		}
+	}
+
+	var contractRevenueDelta types.Currency
+	var activeContractsDelta, failedContractsDelta, successfulContractsDelta, storageUtilizationDelta int64
+	for _, fce := range fces {
+		fc := fce.FileContractElement.FileContract
+		if fce.Revision != nil {
+			fc = fce.Revision.FileContract
+		}
+
+		if fce.Resolved {
+			activeContractsDelta--
+			storageUtilizationDelta -= int64(fc.Filesize)
+		} else if fce.Revision == nil {
+			// don't count revision as a new contract
+			activeContractsDelta++
+			storageUtilizationDelta += int64(fc.Filesize)
+		} else {
+			// filesize changed
+			storageUtilizationDelta += int64(fc.Filesize - fce.FileContractElement.FileContract.Filesize)
+		}
+
+		if fce.Resolved {
+			if !fce.Valid {
+				failedContractsDelta++
+			} else {
+				successfulContractsDelta++
+				for _, vpo := range fc.ValidProofOutputs {
+					contractRevenueDelta = contractRevenueDelta.Add(vpo.Value)
+				}
+			}
+		}
+	}
+
+	var circulatingSupplyDelta types.Currency
+	for _, mp := range b.MinerPayouts {
+		circulatingSupplyDelta = circulatingSupplyDelta.Add(mp.Value)
+	}
+
+	_, err = tx.Exec(`INSERT INTO network_metrics(block_id, height, difficulty, total_hosts, active_contracts, failed_contracts, successful_contracts, storage_utilization, circulating_supply, contract_revenue) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+		encode(b.ID()),
+		height,
+		encode(difficulty),
+		int64(existingMetrics.TotalHosts)+totalHostsDelta,
+		int64(existingMetrics.ActiveContracts)+activeContractsDelta,
+		int64(existingMetrics.FailedContracts)+failedContractsDelta,
+		int64(existingMetrics.SuccessfulContracts)+successfulContractsDelta,
+		int64(existingMetrics.StorageUtilization)+storageUtilizationDelta,
+		encode(existingMetrics.CirculatingSupply.Add(circulatingSupplyDelta)),
+		encode(existingMetrics.ContractRevenue.Add(contractRevenueDelta)),
+	)
+	if err != nil {
+		return fmt.Errorf("addMetrics: failed to execute metrics update query: %w", err)
+	}
+	return nil
+}
+
 func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error {
 	if err := addBlock(ut.tx, state.Block, state.Index.Height); err != nil {
 		return fmt.Errorf("ApplyIndex: failed to add block: %w", err)
@@ -853,6 +947,8 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error {
 		return fmt.Errorf("ApplyIndex: failed to add transactions: addTransactions: %w", err)
 	} else if err := updateStateTree(ut.tx, state.TreeUpdates); err != nil {
 		return fmt.Errorf("ApplyIndex: failed to update state tree: %w", err)
+	} else if err := addMetrics(ut.tx, state.Index.Height, state.Difficulty, state.Block, state.FileContractElements, state.Events); err != nil {
+		return fmt.Errorf("ApplyIndex: failed to update metrics: %w", err)
 	} else if err := addEvents(ut.tx, scDBIds, fcDBIds, txnDBIds, state.Events); err != nil {
 		return fmt.Errorf("ApplyIndex: failed to add events: %w", err)
 	}