Merge pull request #4 from SiaFoundation/store-elements-state

Follow ups

.github/workflows/main.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest , macos-latest, windows-latest ]
-        go-version: [ '1.20', '1.21' ]
+        go-version: [ '1.21', '1.22' ]
     steps:
       - name: Configure git
         run: git config --global core.autocrlf false # required on Windows

api/client.go

@@ -44,12 +44,6 @@ func (c *Client) TxpoolFee() (resp types.Currency, err error) {
 	return
 }
 
-// SyncerPeers returns the current peers of the syncer.
-func (c *Client) SyncerPeers() (resp []GatewayPeer, err error) {
-	err = c.c.GET("/syncer/peers", &resp)
-	return
-}
-
 // SyncerConnect adds the address as a peer of the syncer.
 func (c *Client) SyncerConnect(addr string) (err error) {
 	err = c.c.POST("/syncer/connect", addr, nil)

api/server.go

@@ -10,8 +10,8 @@ import (
 	"go.sia.tech/core/consensus"
 	"go.sia.tech/core/gateway"
 	"go.sia.tech/core/types"
+	"go.sia.tech/coreutils/syncer"
 	"go.sia.tech/explored/explorer"
-	"go.sia.tech/explored/syncer"
 )
 
 type (
@@ -22,20 +22,19 @@ type (
 		RecommendedFee() types.Currency
 		PoolTransactions() []types.Transaction
 		V2PoolTransactions() []types.V2Transaction
-		AddPoolTransactions(txns []types.Transaction) error
-		AddV2PoolTransactions(txns []types.V2Transaction) error
+		AddPoolTransactions(txns []types.Transaction) (bool, error)
+		AddV2PoolTransactions(index types.ChainIndex, txns []types.V2Transaction) (bool, error)
 		UnconfirmedParents(txn types.Transaction) []types.Transaction
 	}
 
 	// A Syncer can connect to other peers and synchronize the blockchain.
 	Syncer interface {
 		Addr() string
-		Peers() []*gateway.Peer
-		PeerInfo(peer string) (syncer.PeerInfo, bool)
-		Connect(addr string) (*gateway.Peer, error)
+		Peers() []*syncer.Peer
+		Connect(addr string) (*syncer.Peer, error)
 		BroadcastHeader(bh gateway.BlockHeader)
 		BroadcastTransactionSet(txns []types.Transaction)
-		BroadcastV2TransactionSet(txns []types.V2Transaction)
+		BroadcastV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction)
 		BroadcastV2BlockOutline(bo gateway.V2BlockOutline)
 	}
 
@@ -57,27 +56,6 @@ type server struct {
 	s  Syncer
 }
 
-func (s *server) syncerPeersHandler(jc jape.Context) {
-	var peers []GatewayPeer
-	for _, p := range s.s.Peers() {
-		info, ok := s.s.PeerInfo(p.Addr)
-		if !ok {
-			continue
-		}
-		peers = append(peers, GatewayPeer{
-			Addr:    p.Addr,
-			Inbound: p.Inbound,
-			Version: p.Version,
-
-			FirstSeen:      info.FirstSeen,
-			ConnectedSince: info.LastConnect,
-			SyncedBlocks:   info.SyncedBlocks,
-			SyncDuration:   info.SyncDuration,
-		})
-	}
-	jc.Encode(peers)
-}
-
 func (s *server) syncerConnectHandler(jc jape.Context) {
 	var addr string
 	if jc.Decode(&addr) != nil {
@@ -122,17 +100,24 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) {
 	if jc.Decode(&tbr) != nil {
 		return
 	}
+
+	tip, err := s.e.Tip()
+	if jc.Check("failed to get tip", err) != nil {
+		return
+	}
 	if len(tbr.Transactions) != 0 {
-		if jc.Check("invalid transaction set", s.cm.AddPoolTransactions(tbr.Transactions)) != nil {
+		_, err := s.cm.AddPoolTransactions(tbr.Transactions)
+		if jc.Check("invalid transaction set", err) != nil {
 			return
 		}
 		s.s.BroadcastTransactionSet(tbr.Transactions)
 	}
 	if len(tbr.V2Transactions) != 0 {
-		if jc.Check("invalid v2 transaction set", s.cm.AddV2PoolTransactions(tbr.V2Transactions)) != nil {
+		_, err := s.cm.AddV2PoolTransactions(tip, tbr.V2Transactions)
+		if jc.Check("invalid v2 transaction set", err) != nil {
 			return
 		}
-		s.s.BroadcastV2TransactionSet(tbr.V2Transactions)
+		s.s.BroadcastV2TransactionSet(tip, tbr.V2Transactions)
 	}
 }
 
@@ -258,7 +243,6 @@ func NewServer(e Explorer, cm ChainManager, s Syncer) http.Handler {
 		s:  s,
 	}
 	return jape.Mux(map[string]jape.Handler{
-		"GET    /syncer/peers":           srv.syncerPeersHandler,
 		"POST   /syncer/connect":         srv.syncerConnectHandler,
 		"POST   /syncer/broadcast/block": srv.syncerBroadcastBlockHandler,
 

cmd/explored/main.go

@@ -75,7 +75,7 @@ func main() {
 	consoleEncoder := zapcore.NewConsoleEncoder(consoleCfg)
 
 	// only log info messages to console unless stdout logging is enabled
-	consoleCore := zapcore.NewCore(consoleEncoder, zapcore.Lock(os.Stdout), zap.NewAtomicLevelAt(zap.InfoLevel))
+	consoleCore := zapcore.NewCore(consoleEncoder, zapcore.Lock(os.Stdout), zap.NewAtomicLevelAt(zap.DebugLevel))
 	log := zap.New(consoleCore, zap.AddCaller())
 	defer log.Sync()
 	// redirect stdlib log to zap

cmd/explored/node.go

@@ -3,21 +3,22 @@ package main
 import (
 	"context"
 	"errors"
-	"log"
+	"io/fs"
 	"net"
+	"os"
 	"path/filepath"
 	"strconv"
 	"time"
 
 	bolt "go.etcd.io/bbolt"
-	"go.sia.tech/core/chain"
 	"go.sia.tech/core/consensus"
 	"go.sia.tech/core/gateway"
 	"go.sia.tech/core/types"
+	"go.sia.tech/coreutils/chain"
+	"go.sia.tech/coreutils/syncer"
 	"go.sia.tech/explored/explorer"
 	"go.sia.tech/explored/internal/syncerutil"
 	"go.sia.tech/explored/persist/sqlite"
-	"go.sia.tech/explored/syncer"
 	"go.uber.org/zap"
 	"lukechampine.com/upnp"
 )
@@ -160,7 +161,7 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool, logger *zap.Lo
 	if err != nil {
 		return nil, err
 	}
-	e := explorer.NewExplorer(store)
+
 	tip, err := store.Tip()
 	if errors.Is(err, sqlite.ErrNoTip) {
 		tip = types.ChainIndex{
@@ -172,6 +173,13 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool, logger *zap.Lo
 	}
 	cm.AddSubscriber(store, tip)
 
+	hashPath := filepath.Join(dir, "./hash")
+	if err := os.MkdirAll(hashPath, fs.ModePerm); err != nil {
+		return nil, err
+	}
+
+	e := explorer.NewExplorer(store)
+
 	l, err := net.Listen("tcp", addr)
 	if err != nil {
 		return nil, err
@@ -218,7 +226,7 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool, logger *zap.Lo
 		UniqueID:   gateway.GenerateUniqueID(),
 		NetAddress: syncerAddr,
 	}
-	s := syncer.New(l, cm, ps, header, syncer.WithLogger(log.Default()))
+	s := syncer.New(l, cm, ps, header, syncer.WithLogger(logger.Named("syncer")))
 
 	return &node{
 		cm: cm,

explorer/explorer.go

@@ -1,8 +1,8 @@
 package explorer
 
 import (
-	"go.sia.tech/core/chain"
 	"go.sia.tech/core/types"
+	"go.sia.tech/coreutils/chain"
 )
 
 // A Store is a database that stores information about elements, contracts,
@@ -17,6 +17,8 @@ type Store interface {
 	UnspentSiacoinOutputs(address types.Address, limit, offset uint64) ([]SiacoinOutput, error)
 	UnspentSiafundOutputs(address types.Address, limit, offset uint64) ([]SiafundOutput, error)
 	Balance(address types.Address) (sc types.Currency, sf uint64, err error)
+
+	MerkleProof(leafIndex uint64) ([]types.Hash256, error)
 }
 
 // Explorer implements a Sia explorer.
@@ -29,6 +31,11 @@ func NewExplorer(s Store) *Explorer {
 	return &Explorer{s: s}
 }
 
+// MerkleProof gets the merkle proof with the given leaf index.
+func (e *Explorer) MerkleProof(leafIndex uint64) ([]types.Hash256, error) {
+	return e.s.MerkleProof(leafIndex)
+}
+
 // Tip returns the tip of the best known valid chain.
 func (e *Explorer) Tip() (types.ChainIndex, error) {
 	return e.s.Tip()

explorer/types.go

@@ -34,24 +34,15 @@ func (d Source) MarshalJSON() ([]byte, error) {
 	}
 }
 
-// A SiacoinOutput is a types.SiacoinOutput with added fields for output ID,
-// source, and maturity height.
+// A SiacoinOutput is a types.SiacoinElement with an added field for the
+// source.
 type SiacoinOutput struct {
-	OutputID       types.SiacoinOutputID `json:"outputID"`
-	Source         Source                `json:"source"`
-	MaturityHeight int                   `json:"maturityHeight"`
-
-	types.SiacoinOutput
+	Source Source `json:"source"`
+	types.SiacoinElement
 }
 
-// A SiafundOutput contains a types.SiafundOutput with added fields for output
-// ID and claim start.
-type SiafundOutput struct {
-	OutputID   types.SiafundOutputID `json:"outputID"`
-	ClaimStart types.Currency        `json:"claimStart"`
-
-	types.SiafundOutput
-}
+// A SiafundOutput is a types.SiafundElement.
+type SiafundOutput types.SiafundElement
 
 // A Transaction is a transaction that uses the wrapped types above.
 type Transaction struct {

go.mod

@@ -1,11 +1,12 @@
 module go.sia.tech/explored
 
-go 1.18
+go 1.21.6
 
 require (
 	github.com/mattn/go-sqlite3 v1.14.19
-	go.etcd.io/bbolt v1.3.7
-	go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0
+	go.etcd.io/bbolt v1.3.8
+	go.sia.tech/core v0.2.1
+	go.sia.tech/coreutils v0.0.2
 	go.sia.tech/jape v0.11.1
 	go.uber.org/zap v1.26.0
 	golang.org/x/term v0.6.0

go.sum

@@ -1,30 +1,35 @@
 github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY=
 github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
 github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
-go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
-go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
-go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0 h1:2nKOKa99g9h9m3hL5UortAbmnwuwXhDcTHIhzmqBae8=
-go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q=
-go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY=
-go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
+go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.sia.tech/core v0.2.1 h1:CqmMd+T5rAhC+Py3NxfvGtvsj/GgwIqQHHVrdts/LqY=
+go.sia.tech/core v0.2.1/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q=
+go.sia.tech/coreutils v0.0.2 h1:vDqMDM6dW6b/R3sO1ycr8fAnJXUiAvrzxehEIq/AsKA=
+go.sia.tech/coreutils v0.0.2/go.mod h1:UBFc77wXiE//eyilO5HLOncIEj7F69j0Nv2OkFujtP0=
 go.sia.tech/jape v0.11.1 h1:M7IP+byXL7xOqzxcHUQuXW+q3sYMkYzmMlMw+q8ZZw0=
 go.sia.tech/jape v0.11.1/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4=
 go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU=
 go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo=
 go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
+go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
 go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
 go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
 go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
 golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 h1:NvGWuYG8dkDHFSKksI1P9faiVJ9rayE6l0+ouWVIDs8=
 golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs=
+golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -33,6 +38,7 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=
 golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw=
 lukechampine.com/frand v1.4.2/go.mod h1:4S/TM2ZgrKejMcKMbeLjISpJMO+/eZ1zu3vYX9dtj3s=
 lukechampine.com/upnp v0.3.0 h1:UVCD6eD6fmJmwak6DVE3vGN+L46Fk8edTcC6XYCb6C4=