From b6aec5ba5acc230018b6e31aa4f0e77d47be4aaf Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Fri, 22 Dec 2023 19:22:40 -0500 Subject: [PATCH 01/18] add blocks and miner payouts to database --- cmd/explored/node.go | 1 + explorer/explorer.go | 23 +++++++++++++++++++--- persist/sqlite/init.sql | 16 +++++++++++++++ persist/sqlite/txn.go | 43 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 persist/sqlite/txn.go diff --git a/cmd/explored/node.go b/cmd/explored/node.go index 3890958c..a3714ca8 100644 --- a/cmd/explored/node.go +++ b/cmd/explored/node.go @@ -161,6 +161,7 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool, logger *zap.Lo panic(err) } e := explorer.NewExplorer(store) + cm.AddSubscriber(e, cm.Tip()) l, err := net.Listen("tcp", addr) if err != nil { diff --git a/explorer/explorer.go b/explorer/explorer.go index 64af30b1..d155e336 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -1,10 +1,20 @@ package explorer -import "go.sia.tech/core/chain" +import ( + "go.sia.tech/core/chain" + "go.sia.tech/core/types" +) + +type Transaction interface { + AddBlock(b types.Block, height uint64) error + AddMinerPayouts(bid types.BlockID, scos []types.SiacoinOutput) error +} // A Store is a database that stores information about elements, contracts, // and blocks. -type Store interface{} +type Store interface { + Transaction(fn func(tx Transaction) error) error +} // Explorer implements a Sia explorer. type Explorer struct { @@ -18,7 +28,14 @@ func NewExplorer(s Store) *Explorer { // ProcessChainApplyUpdate implements chain.Subscriber. func (e *Explorer) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) error { - return nil + return e.s.Transaction(func(tx Transaction) error { + if err := tx.AddBlock(cau.Block, cau.State.Index.Height); err != nil { + return err + } else if err := tx.AddMinerPayouts(cau.Block.ID(), cau.Block.MinerPayouts); err != nil { + return err + } + return nil + }) } // ProcessChainRevertUpdate implements chain.Subscriber. diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index b80d5b7d..8893b3a8 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -3,5 +3,21 @@ CREATE TABLE global_settings ( db_version INTEGER NOT NULL -- used for migrations ); +CREATE TABLE Blocks ( + id BINARY(32) NOT NULL PRIMARY KEY, + height INTEGER NOT NULL, + parent_id BINARY(32) NOT NULL, + nonce INTEGER NOT NULL, + timestamp INTEGER NOT NULL +); + +CREATE TABLE MinerPayouts ( + block_id REFERENCES Blocks(id), + block_order INTEGER NOT NULL, + address BINARY(32) NOT NULL, + value BINARY(16) NOT NULL +); + + -- initialize the global settings table INSERT INTO global_settings (id, db_version) VALUES (0, 0); -- should not be changed diff --git a/persist/sqlite/txn.go b/persist/sqlite/txn.go new file mode 100644 index 00000000..2f15dadd --- /dev/null +++ b/persist/sqlite/txn.go @@ -0,0 +1,43 @@ +package sqlite + +import ( + "bytes" + + "go.sia.tech/core/types" + "go.sia.tech/explored/explorer" +) + +type explorerTxn struct { + tx txn +} + +// Transaction implements explorer.Store. +func (s *Store) Transaction(fn func(tx explorer.Transaction) error) error { + return s.transaction(func(tx txn) error { + return fn(&explorerTxn{tx: tx}) + }) +} + +func encode(obj types.EncoderTo) []byte { + var buf bytes.Buffer + e := types.NewEncoder(&buf) + obj.EncodeTo(e) + e.Flush() + return buf.Bytes() +} + +// AddBlock implements explorer.Transaction. +func (tx *explorerTxn) AddBlock(b types.Block, height uint64) error { + _, err := tx.tx.Exec("INSERT INTO Blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), b.Nonce, b.Timestamp.Unix()) + return err +} + +// AddMinerPayouts implements explorer.Transaction. +func (tx *explorerTxn) AddMinerPayouts(bid types.BlockID, scos []types.SiacoinOutput) error { + for i, sco := range scos { + if _, err := tx.tx.Exec("INSERT INTO MinerPayouts(block_id, block_order, address, value) VALUES (?, ?, ?, ?);", encode(bid), i, encode(sco.Address), encode(sco.Value)); err != nil { + return err + } + } + return nil +} From 0c0d667ac18ee9c1852abb785e3659b9d53e893f Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Tue, 2 Jan 2024 13:37:08 -0500 Subject: [PATCH 02/18] store multiple applyupdates and commit batches at once --- cmd/explored/node.go | 1 + explorer/explorer.go | 51 +++++++++++++++++++++++++++++++++-------- persist/sqlite/init.sql | 4 ++-- persist/sqlite/txn.go | 16 ++++++++++++- 4 files changed, 60 insertions(+), 12 deletions(-) diff --git a/cmd/explored/node.go b/cmd/explored/node.go index a3714ca8..f24a3bcc 100644 --- a/cmd/explored/node.go +++ b/cmd/explored/node.go @@ -161,6 +161,7 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool, logger *zap.Lo panic(err) } e := explorer.NewExplorer(store) + // eventually make the tip equal the latest block in the DB cm.AddSubscriber(e, cm.Tip()) l, err := net.Listen("tcp", addr) diff --git a/explorer/explorer.go b/explorer/explorer.go index d155e336..76895bdb 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -1,13 +1,18 @@ package explorer import ( + "sync" + "go.sia.tech/core/chain" "go.sia.tech/core/types" ) type Transaction interface { + // Create AddBlock(b types.Block, height uint64) error AddMinerPayouts(bid types.BlockID, scos []types.SiacoinOutput) error + // Delete + DeleteBlock(bid types.BlockID) error } // A Store is a database that stores information about elements, contracts, @@ -18,27 +23,55 @@ type Store interface { // Explorer implements a Sia explorer. type Explorer struct { - s Store + mu sync.Mutex + + s Store + pendingUpdates []*chain.ApplyUpdate } // NewExplorer returns a Sia explorer. func NewExplorer(s Store) *Explorer { - return &Explorer{s} + return &Explorer{s: s} +} + +func (e *Explorer) applyUpdates(tx Transaction) error { + for _, update := range e.pendingUpdates { + if err := tx.AddBlock(update.Block, update.State.Index.Height); err != nil { + return err + } else if err := tx.AddMinerPayouts(update.Block.ID(), update.Block.MinerPayouts); err != nil { + return err + } + } + e.pendingUpdates = e.pendingUpdates[:0] + return nil } // ProcessChainApplyUpdate implements chain.Subscriber. func (e *Explorer) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) error { + e.mu.Lock() + defer e.mu.Unlock() + + e.pendingUpdates = append(e.pendingUpdates, cau) + if mayCommit { + return e.s.Transaction(e.applyUpdates) + } + return nil +} + +// ProcessChainRevertUpdate implements chain.Subscriber. +func (e *Explorer) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { + e.mu.Lock() + defer e.mu.Unlock() + return e.s.Transaction(func(tx Transaction) error { - if err := tx.AddBlock(cau.Block, cau.State.Index.Height); err != nil { + if err := e.applyUpdates(tx); err != nil { + panic(err) return err - } else if err := tx.AddMinerPayouts(cau.Block.ID(), cau.Block.MinerPayouts); err != nil { + } + if err := tx.DeleteBlock(cru.Block.ID()); err != nil { + panic(err) return err } return nil }) } - -// ProcessChainRevertUpdate implements chain.Subscriber. -func (e *Explorer) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { - return nil -} diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 8893b3a8..97ff509d 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -7,12 +7,12 @@ CREATE TABLE Blocks ( id BINARY(32) NOT NULL PRIMARY KEY, height INTEGER NOT NULL, parent_id BINARY(32) NOT NULL, - nonce INTEGER NOT NULL, + nonce BINARY(8) NOT NULL, timestamp INTEGER NOT NULL ); CREATE TABLE MinerPayouts ( - block_id REFERENCES Blocks(id), + block_id REFERENCES Blocks(id) ON DELETE CASCADE, block_order INTEGER NOT NULL, address BINARY(32) NOT NULL, value BINARY(16) NOT NULL diff --git a/persist/sqlite/txn.go b/persist/sqlite/txn.go index 2f15dadd..0011b3f6 100644 --- a/persist/sqlite/txn.go +++ b/persist/sqlite/txn.go @@ -26,9 +26,17 @@ func encode(obj types.EncoderTo) []byte { return buf.Bytes() } +func encodeUint64(x uint64) []byte { + var buf bytes.Buffer + e := types.NewEncoder(&buf) + e.WriteUint64(x) + e.Flush() + return buf.Bytes() +} + // AddBlock implements explorer.Transaction. func (tx *explorerTxn) AddBlock(b types.Block, height uint64) error { - _, err := tx.tx.Exec("INSERT INTO Blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), b.Nonce, b.Timestamp.Unix()) + _, err := tx.tx.Exec("INSERT INTO Blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encodeUint64(b.Nonce), b.Timestamp.Unix()) return err } @@ -41,3 +49,9 @@ func (tx *explorerTxn) AddMinerPayouts(bid types.BlockID, scos []types.SiacoinOu } return nil } + +// DeleteBlock implements explorer.Transaction. +func (tx *explorerTxn) DeleteBlock(bid types.BlockID) error { + _, err := tx.tx.Exec("DELETE FROM Blocks WHERE id = ?", encode(bid)) + return err +} From 51a704345f8161941a0933899b27cacb04320ed8 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Wed, 3 Jan 2024 12:53:35 -0500 Subject: [PATCH 03/18] move chain.Subscriber implementation to Store --- cmd/explored/node.go | 2 +- explorer/explorer.go | 60 ++---------------------------------- persist/sqlite/store.go | 5 +++ persist/sqlite/txn.go | 67 ++++++++++++++++++++++++++++------------- 4 files changed, 54 insertions(+), 80 deletions(-) diff --git a/cmd/explored/node.go b/cmd/explored/node.go index f24a3bcc..f6d67439 100644 --- a/cmd/explored/node.go +++ b/cmd/explored/node.go @@ -162,7 +162,7 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool, logger *zap.Lo } e := explorer.NewExplorer(store) // eventually make the tip equal the latest block in the DB - cm.AddSubscriber(e, cm.Tip()) + cm.AddSubscriber(store, cm.Tip()) l, err := net.Listen("tcp", addr) if err != nil { diff --git a/explorer/explorer.go b/explorer/explorer.go index 76895bdb..94626f43 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -1,77 +1,21 @@ package explorer import ( - "sync" - "go.sia.tech/core/chain" - "go.sia.tech/core/types" ) -type Transaction interface { - // Create - AddBlock(b types.Block, height uint64) error - AddMinerPayouts(bid types.BlockID, scos []types.SiacoinOutput) error - // Delete - DeleteBlock(bid types.BlockID) error -} - // A Store is a database that stores information about elements, contracts, // and blocks. type Store interface { - Transaction(fn func(tx Transaction) error) error + chain.Subscriber } // Explorer implements a Sia explorer. type Explorer struct { - mu sync.Mutex - - s Store - pendingUpdates []*chain.ApplyUpdate + s Store } // NewExplorer returns a Sia explorer. func NewExplorer(s Store) *Explorer { return &Explorer{s: s} } - -func (e *Explorer) applyUpdates(tx Transaction) error { - for _, update := range e.pendingUpdates { - if err := tx.AddBlock(update.Block, update.State.Index.Height); err != nil { - return err - } else if err := tx.AddMinerPayouts(update.Block.ID(), update.Block.MinerPayouts); err != nil { - return err - } - } - e.pendingUpdates = e.pendingUpdates[:0] - return nil -} - -// ProcessChainApplyUpdate implements chain.Subscriber. -func (e *Explorer) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) error { - e.mu.Lock() - defer e.mu.Unlock() - - e.pendingUpdates = append(e.pendingUpdates, cau) - if mayCommit { - return e.s.Transaction(e.applyUpdates) - } - return nil -} - -// ProcessChainRevertUpdate implements chain.Subscriber. -func (e *Explorer) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { - e.mu.Lock() - defer e.mu.Unlock() - - return e.s.Transaction(func(tx Transaction) error { - if err := e.applyUpdates(tx); err != nil { - panic(err) - return err - } - if err := tx.DeleteBlock(cru.Block.ID()); err != nil { - panic(err) - return err - } - return nil - }) -} diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index 458b2cc2..a9671e14 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -7,9 +7,11 @@ import ( "fmt" "math" "strings" + "sync" "time" "github.com/mattn/go-sqlite3" + "go.sia.tech/core/chain" "go.uber.org/zap" "lukechampine.com/frand" ) @@ -19,6 +21,9 @@ type ( Store struct { db *sql.DB log *zap.Logger + + mu sync.Mutex + pendingUpdates []*chain.ApplyUpdate } ) diff --git a/persist/sqlite/txn.go b/persist/sqlite/txn.go index 0011b3f6..1d7ae40e 100644 --- a/persist/sqlite/txn.go +++ b/persist/sqlite/txn.go @@ -3,21 +3,10 @@ package sqlite import ( "bytes" + "go.sia.tech/core/chain" "go.sia.tech/core/types" - "go.sia.tech/explored/explorer" ) -type explorerTxn struct { - tx txn -} - -// Transaction implements explorer.Store. -func (s *Store) Transaction(fn func(tx explorer.Transaction) error) error { - return s.transaction(func(tx txn) error { - return fn(&explorerTxn{tx: tx}) - }) -} - func encode(obj types.EncoderTo) []byte { var buf bytes.Buffer e := types.NewEncoder(&buf) @@ -34,24 +23,60 @@ func encodeUint64(x uint64) []byte { return buf.Bytes() } -// AddBlock implements explorer.Transaction. -func (tx *explorerTxn) AddBlock(b types.Block, height uint64) error { - _, err := tx.tx.Exec("INSERT INTO Blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encodeUint64(b.Nonce), b.Timestamp.Unix()) +func (s *Store) addBlock(b types.Block, height uint64) error { + _, err := s.exec("INSERT INTO Blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encodeUint64(b.Nonce), b.Timestamp.Unix()) return err } -// AddMinerPayouts implements explorer.Transaction. -func (tx *explorerTxn) AddMinerPayouts(bid types.BlockID, scos []types.SiacoinOutput) error { +func (s *Store) addMinerPayouts(bid types.BlockID, scos []types.SiacoinOutput) error { for i, sco := range scos { - if _, err := tx.tx.Exec("INSERT INTO MinerPayouts(block_id, block_order, address, value) VALUES (?, ?, ?, ?);", encode(bid), i, encode(sco.Address), encode(sco.Value)); err != nil { + if _, err := s.exec("INSERT INTO MinerPayouts(block_id, block_order, address, value) VALUES (?, ?, ?, ?);", encode(bid), i, encode(sco.Address), encode(sco.Value)); err != nil { return err } } return nil } -// DeleteBlock implements explorer.Transaction. -func (tx *explorerTxn) DeleteBlock(bid types.BlockID) error { - _, err := tx.tx.Exec("DELETE FROM Blocks WHERE id = ?", encode(bid)) +func (s *Store) deleteBlock(bid types.BlockID) error { + _, err := s.exec("DELETE FROM Blocks WHERE id = ?", encode(bid)) return err } + +func (s *Store) applyUpdates() error { + for _, update := range s.pendingUpdates { + if err := s.addBlock(update.Block, update.State.Index.Height); err != nil { + return err + } else if err := s.addMinerPayouts(update.Block.ID(), update.Block.MinerPayouts); err != nil { + return err + } + } + s.pendingUpdates = s.pendingUpdates[:0] + return nil +} + +// ProcessChainApplyUpdate implements chain.Subscriber. +func (s *Store) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) error { + s.mu.Lock() + defer s.mu.Unlock() + + s.pendingUpdates = append(s.pendingUpdates, cau) + if mayCommit { + return s.applyUpdates() + } + return nil +} + +// ProcessChainRevertUpdate implements chain.Subscriber. +func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { + s.mu.Lock() + defer s.mu.Unlock() + + if err := s.applyUpdates(); err != nil { + return err + } + if err := s.deleteBlock(cru.Block.ID()); err != nil { + return err + } + + return nil +} From 6ad34fa5c179394e4aff8e58a7bde84b43482bf2 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Wed, 3 Jan 2024 13:30:25 -0500 Subject: [PATCH 04/18] properly set tip for subscriber --- cmd/explored/node.go | 12 +++++++++--- explorer/explorer.go | 3 +++ persist/sqlite/query.go | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 persist/sqlite/query.go diff --git a/cmd/explored/node.go b/cmd/explored/node.go index f6d67439..c8f07e7b 100644 --- a/cmd/explored/node.go +++ b/cmd/explored/node.go @@ -158,11 +158,17 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool, logger *zap.Lo store, err := sqlite.OpenDatabase("./explore.db", logger) if err != nil { - panic(err) + return nil, err } e := explorer.NewExplorer(store) - // eventually make the tip equal the latest block in the DB - cm.AddSubscriber(store, cm.Tip()) + tip, err := store.Tip() + if err != nil { + tip = types.ChainIndex{ + ID: genesisBlock.ID(), + Height: 0, + } + } + cm.AddSubscriber(store, tip) l, err := net.Listen("tcp", addr) if err != nil { diff --git a/explorer/explorer.go b/explorer/explorer.go index 94626f43..3d06b97a 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -2,12 +2,15 @@ package explorer import ( "go.sia.tech/core/chain" + "go.sia.tech/core/types" ) // A Store is a database that stores information about elements, contracts, // and blocks. type Store interface { chain.Subscriber + + Tip() (types.ChainIndex, error) } // Explorer implements a Sia explorer. diff --git a/persist/sqlite/query.go b/persist/sqlite/query.go new file mode 100644 index 00000000..f49837f9 --- /dev/null +++ b/persist/sqlite/query.go @@ -0,0 +1,18 @@ +package sqlite + +import "go.sia.tech/core/types" + +func decode(obj types.DecoderFrom, data []byte) error { + d := types.NewBufDecoder(data) + obj.DecodeFrom(d) + return d.Err() +} + +func (s *Store) Tip() (result types.ChainIndex, err error) { + var data []byte + if err = s.queryRow("SELECT id, height FROM Blocks WHERE height = (SELECT MAX(height) from Blocks)").Scan(&data, &result.Height); err != nil { + return + } + decode(&result.ID, data) + return +} From 60bc47d8db941abd4e7b1e8aa21a1baacb7e6d56 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Wed, 3 Jan 2024 13:54:13 -0500 Subject: [PATCH 05/18] add basic explorer API endpoint --- api/client.go | 6 ++++++ api/server.go | 17 ++++++++++++++++- cmd/explored/web.go | 2 +- explorer/explorer.go | 4 ++++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/api/client.go b/api/client.go index ac97d73d..ebeb9c84 100644 --- a/api/client.go +++ b/api/client.go @@ -58,3 +58,9 @@ func (c *Client) SyncerBroadcastBlock(b types.Block) (err error) { err = c.c.POST("/syncer/broadcast/block", b, nil) return } + +// Tip returns the current tip of the explorer. +func (c *Client) Tip() (resp types.ChainIndex, err error) { + err = c.c.GET("/explorer/tip", &resp) + return +} diff --git a/api/server.go b/api/server.go index a3be7682..1fe97531 100644 --- a/api/server.go +++ b/api/server.go @@ -36,10 +36,16 @@ type ( BroadcastV2TransactionSet(txns []types.V2Transaction) BroadcastV2BlockOutline(bo gateway.V2BlockOutline) } + + // Explorer implements a Sia explorer. + Explorer interface { + Tip() (types.ChainIndex, error) + } ) type server struct { cm ChainManager + e Explorer s Syncer mu sync.Mutex @@ -124,10 +130,17 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { } } +func (s *server) explorerTipHandler(jc jape.Context) { + tip, err := s.e.Tip() + jc.Check("failed to get tip", err) + jc.Encode(tip) +} + // NewServer returns an HTTP handler that serves the explored API. -func NewServer(cm ChainManager, s Syncer) http.Handler { +func NewServer(e Explorer, cm ChainManager, s Syncer) http.Handler { srv := server{ cm: cm, + e: e, s: s, } return jape.Mux(map[string]jape.Handler{ @@ -138,5 +151,7 @@ func NewServer(cm ChainManager, s Syncer) http.Handler { "GET /txpool/transactions": srv.txpoolTransactionsHandler, "GET /txpool/fee": srv.txpoolFeeHandler, "POST /txpool/broadcast": srv.txpoolBroadcastHandler, + + "GET /explorer/tip": srv.explorerTipHandler, }) } diff --git a/cmd/explored/web.go b/cmd/explored/web.go index fbaab8e4..c4bc646e 100644 --- a/cmd/explored/web.go +++ b/cmd/explored/web.go @@ -10,7 +10,7 @@ import ( ) func startWeb(l net.Listener, node *node, password string) error { - renter := api.NewServer(node.cm, node.s) + renter := api.NewServer(node.e, node.cm, node.s) api := jape.BasicAuth(password)(renter) return http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(r.URL.Path, "/api") { diff --git a/explorer/explorer.go b/explorer/explorer.go index 3d06b97a..f0883788 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -22,3 +22,7 @@ type Explorer struct { func NewExplorer(s Store) *Explorer { return &Explorer{s: s} } + +func (e *Explorer) Tip() (types.ChainIndex, error) { + return e.s.Tip() +} From 3d189c76b8ee6a463fd04f5f238784e875a549c1 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Wed, 3 Jan 2024 14:45:50 -0500 Subject: [PATCH 06/18] add /explorer/block/:id endpoint --- api/client.go | 8 ++++++ api/server.go | 20 ++++++++++++-- explorer/explorer.go | 5 ++++ persist/sqlite/query.go | 61 +++++++++++++++++++++++++++++++++++++++-- persist/sqlite/txn.go | 1 + 5 files changed, 91 insertions(+), 4 deletions(-) diff --git a/api/client.go b/api/client.go index ebeb9c84..7e84bd29 100644 --- a/api/client.go +++ b/api/client.go @@ -1,6 +1,8 @@ package api import ( + "fmt" + "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/jape" @@ -64,3 +66,9 @@ func (c *Client) Tip() (resp types.ChainIndex, err error) { err = c.c.GET("/explorer/tip", &resp) return } + +// Block returns a block with the given block ID. +func (c *Client) Block(id types.BlockID) (resp types.Block, err error) { + err = c.c.GET(fmt.Sprintf("/explorer/block/%s", id), &resp) + return +} diff --git a/api/server.go b/api/server.go index 1fe97531..febaca75 100644 --- a/api/server.go +++ b/api/server.go @@ -40,6 +40,7 @@ type ( // Explorer implements a Sia explorer. Explorer interface { Tip() (types.ChainIndex, error) + Block(id types.BlockID) (types.Block, error) } ) @@ -132,10 +133,24 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { func (s *server) explorerTipHandler(jc jape.Context) { tip, err := s.e.Tip() - jc.Check("failed to get tip", err) + if jc.Check("failed to get tip", err) != nil { + return + } jc.Encode(tip) } +func (s *server) explorerBlockHandler(jc jape.Context) { + var id types.BlockID + if jc.DecodeParam("id", &id) != nil { + return + } + block, err := s.e.Block(id) + if jc.Check("failed to get block", err) != nil { + return + } + jc.Encode(block) +} + // NewServer returns an HTTP handler that serves the explored API. func NewServer(e Explorer, cm ChainManager, s Syncer) http.Handler { srv := server{ @@ -152,6 +167,7 @@ func NewServer(e Explorer, cm ChainManager, s Syncer) http.Handler { "GET /txpool/fee": srv.txpoolFeeHandler, "POST /txpool/broadcast": srv.txpoolBroadcastHandler, - "GET /explorer/tip": srv.explorerTipHandler, + "GET /explorer/tip": srv.explorerTipHandler, + "GET /explorer/block/:id": srv.explorerBlockHandler, }) } diff --git a/explorer/explorer.go b/explorer/explorer.go index f0883788..78f3499b 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -11,6 +11,7 @@ type Store interface { chain.Subscriber Tip() (types.ChainIndex, error) + Block(id types.BlockID) (types.Block, error) } // Explorer implements a Sia explorer. @@ -26,3 +27,7 @@ func NewExplorer(s Store) *Explorer { func (e *Explorer) Tip() (types.ChainIndex, error) { return e.s.Tip() } + +func (e *Explorer) Block(id types.BlockID) (types.Block, error) { + return e.s.Block(id) +} diff --git a/persist/sqlite/query.go b/persist/sqlite/query.go index f49837f9..8e5648d2 100644 --- a/persist/sqlite/query.go +++ b/persist/sqlite/query.go @@ -1,6 +1,10 @@ package sqlite -import "go.sia.tech/core/types" +import ( + "time" + + "go.sia.tech/core/types" +) func decode(obj types.DecoderFrom, data []byte) error { d := types.NewBufDecoder(data) @@ -8,11 +12,64 @@ func decode(obj types.DecoderFrom, data []byte) error { return d.Err() } +func decodeUint64(x *uint64, data []byte) error { + d := types.NewBufDecoder(data) + if x != nil { + *x = d.ReadUint64() + } + return d.Err() +} + +// Tip implements explorer.Store. func (s *Store) Tip() (result types.ChainIndex, err error) { var data []byte if err = s.queryRow("SELECT id, height FROM Blocks WHERE height = (SELECT MAX(height) from Blocks)").Scan(&data, &result.Height); err != nil { return } - decode(&result.ID, data) + if err = decode(&result.ID, data); err != nil { + return + } + return +} + +// Block implements explorer.Store. +func (s *Store) Block(id types.BlockID) (result types.Block, err error) { + { + var timestamp int64 + var parentID, nonce []byte + if err = s.queryRow("SELECT parent_id, nonce, timestamp FROM Blocks WHERE id = ?", encode(id)).Scan(&parentID, &nonce, ×tamp); err != nil { + return + } + result.Timestamp = time.Unix(timestamp, 0).UTC() + if err = decode(&result.ParentID, parentID); err != nil { + return + } + if err = decodeUint64(&result.Nonce, nonce); err != nil { + return + } + } + + { + var rows *loggedRows + if rows, err = s.query("SELECT address, value FROM MinerPayouts WHERE block_id = ? ORDER BY block_order", encode(id)); err != nil { + return + } + + var address, value []byte + for rows.Next() { + if err = rows.Scan(&address, &value); err != nil { + return + } + var minerPayout types.SiacoinOutput + if err = decode(&minerPayout.Address, address); err != nil { + return + } + if err = decode(&minerPayout.Value, value); err != nil { + return + } + result.MinerPayouts = append(result.MinerPayouts, minerPayout) + } + } + return } diff --git a/persist/sqlite/txn.go b/persist/sqlite/txn.go index 1d7ae40e..fba3b3f7 100644 --- a/persist/sqlite/txn.go +++ b/persist/sqlite/txn.go @@ -24,6 +24,7 @@ func encodeUint64(x uint64) []byte { } func (s *Store) addBlock(b types.Block, height uint64) error { + // nonce is encoded because database/sql doesn't support uint64 with high bit set _, err := s.exec("INSERT INTO Blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encodeUint64(b.Nonce), b.Timestamp.Unix()) return err } From bb8ebca75e50e96b95684e2e21af9c6b281c009e Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Wed, 3 Jan 2024 14:50:01 -0500 Subject: [PATCH 07/18] add documentation and fix lint warning --- explorer/explorer.go | 2 ++ persist/sqlite/txn.go | 6 +----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/explorer/explorer.go b/explorer/explorer.go index 78f3499b..bf7cd592 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -24,10 +24,12 @@ func NewExplorer(s Store) *Explorer { return &Explorer{s: s} } +// Tip returns the tip of the best known valid chain. func (e *Explorer) Tip() (types.ChainIndex, error) { return e.s.Tip() } +// Block returns the block with the specified ID. func (e *Explorer) Block(id types.BlockID) (types.Block, error) { return e.s.Block(id) } diff --git a/persist/sqlite/txn.go b/persist/sqlite/txn.go index fba3b3f7..f091a6c8 100644 --- a/persist/sqlite/txn.go +++ b/persist/sqlite/txn.go @@ -75,9 +75,5 @@ func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { if err := s.applyUpdates(); err != nil { return err } - if err := s.deleteBlock(cru.Block.ID()); err != nil { - return err - } - - return nil + return s.deleteBlock(cru.Block.ID()) } From 4997e2eedbabe09ce459a3d995b8e56a01ceaa6f Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Thu, 4 Jan 2024 12:02:58 -0500 Subject: [PATCH 08/18] add endpoint to get block by height --- api/client.go | 10 ++++++++-- api/server.go | 18 ++++++++++++++++-- explorer/explorer.go | 6 ++++++ go.mod | 4 ++-- go.sum | 3 +++ persist/sqlite/query.go | 15 +++++++++++++++ 6 files changed, 50 insertions(+), 6 deletions(-) diff --git a/api/client.go b/api/client.go index 7e84bd29..e76ee7f5 100644 --- a/api/client.go +++ b/api/client.go @@ -67,8 +67,14 @@ func (c *Client) Tip() (resp types.ChainIndex, err error) { return } -// Block returns a block with the given block ID. +// Block returns the block with the specified ID. func (c *Client) Block(id types.BlockID) (resp types.Block, err error) { - err = c.c.GET(fmt.Sprintf("/explorer/block/%s", id), &resp) + err = c.c.GET(fmt.Sprintf("/explorer/block/id/%s", id), &resp) + return +} + +// BlockHeight returns the block with the specified height. +func (c *Client) BlockHeight(height uint64) (resp types.Block, err error) { + err = c.c.GET(fmt.Sprintf("/explorer/block/height/%d", height), &resp) return } diff --git a/api/server.go b/api/server.go index febaca75..6549338e 100644 --- a/api/server.go +++ b/api/server.go @@ -41,6 +41,7 @@ type ( Explorer interface { Tip() (types.ChainIndex, error) Block(id types.BlockID) (types.Block, error) + BlockHeight(height uint64) (types.Block, error) } ) @@ -151,6 +152,18 @@ func (s *server) explorerBlockHandler(jc jape.Context) { jc.Encode(block) } +func (s *server) explorerBlockHeightHandler(jc jape.Context) { + var height uint64 + if jc.DecodeParam("height", &height) != nil { + return + } + block, err := s.e.BlockHeight(height) + if jc.Check("failed to get block", err) != nil { + return + } + jc.Encode(block) +} + // NewServer returns an HTTP handler that serves the explored API. func NewServer(e Explorer, cm ChainManager, s Syncer) http.Handler { srv := server{ @@ -167,7 +180,8 @@ func NewServer(e Explorer, cm ChainManager, s Syncer) http.Handler { "GET /txpool/fee": srv.txpoolFeeHandler, "POST /txpool/broadcast": srv.txpoolBroadcastHandler, - "GET /explorer/tip": srv.explorerTipHandler, - "GET /explorer/block/:id": srv.explorerBlockHandler, + "GET /explorer/tip": srv.explorerTipHandler, + "GET /explorer/block/id/:id": srv.explorerBlockHandler, + "GET /explorer/block/height/:height": srv.explorerBlockHeightHandler, }) } diff --git a/explorer/explorer.go b/explorer/explorer.go index bf7cd592..f45b7b2b 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -12,6 +12,7 @@ type Store interface { Tip() (types.ChainIndex, error) Block(id types.BlockID) (types.Block, error) + BlockHeight(height uint64) (types.Block, error) } // Explorer implements a Sia explorer. @@ -33,3 +34,8 @@ func (e *Explorer) Tip() (types.ChainIndex, error) { func (e *Explorer) Block(id types.BlockID) (types.Block, error) { return e.s.Block(id) } + +// BlockHeight returns the block with the specified height. +func (e *Explorer) BlockHeight(height uint64) (types.Block, error) { + return e.s.BlockHeight(height) +} diff --git a/go.mod b/go.mod index e5829bdb..01cca0dc 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,8 @@ require ( github.com/mattn/go-sqlite3 v1.14.19 go.etcd.io/bbolt v1.3.7 go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0 - go.sia.tech/jape v0.9.0 + go.sia.tech/jape v0.11.1 + go.uber.org/zap v1.26.0 golang.org/x/term v0.6.0 lukechampine.com/frand v1.4.2 lukechampine.com/upnp v0.3.0 @@ -17,7 +18,6 @@ require ( github.com/julienschmidt/httprouter v1.3.0 // indirect go.sia.tech/mux v1.2.0 // indirect go.uber.org/multierr v1.10.0 // indirect - go.uber.org/zap v1.26.0 // indirect golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 // indirect golang.org/x/sys v0.6.0 // indirect golang.org/x/tools v0.7.0 // indirect diff --git a/go.sum b/go.sum index d513a318..5ef5bcbb 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,11 @@ go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0 h1:2nKOKa99g9h9m3hL5UortA go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= +go.sia.tech/jape v0.11.1 h1:M7IP+byXL7xOqzxcHUQuXW+q3sYMkYzmMlMw+q8ZZw0= +go.sia.tech/jape v0.11.1/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= diff --git a/persist/sqlite/query.go b/persist/sqlite/query.go index 8e5648d2..a94d25eb 100644 --- a/persist/sqlite/query.go +++ b/persist/sqlite/query.go @@ -73,3 +73,18 @@ func (s *Store) Block(id types.BlockID) (result types.Block, err error) { return } + +// BlockHeight implements explorer.Store. +func (s *Store) BlockHeight(height uint64) (result types.Block, err error) { + var data []byte + if err = s.queryRow("SELECT id FROM Blocks WHERE height = ?", height).Scan(&data); err != nil { + return + } + + var bid types.BlockID + if err = decode(&bid, data); err != nil { + return + } + result, err = s.Block(bid) + return +} From 2819390cbb6df278a05e6c3ffe876ca6d92dfba2 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Thu, 4 Jan 2024 12:26:25 -0500 Subject: [PATCH 09/18] try to fix windows build --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index af724e8a..1350447c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -25,4 +25,4 @@ jobs: - name: Test uses: ./.github/actions/test - name: Build - run: go build -o bin/ ./cmd/explored + run: CGO_ENABLED=1 go install github.com/mattn/go-sqlite3 && go build -o bin/ ./cmd/explored From faa3507befc7e0c1f50bb51fe0361ba0c2ebfd25 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Thu, 4 Jan 2024 12:40:58 -0500 Subject: [PATCH 10/18] fix syntax --- .github/workflows/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1350447c..31a8c86f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,6 +5,8 @@ on: push: branches: - master +env: + CGO_ENABLED: 1 jobs: test: @@ -25,4 +27,6 @@ jobs: - name: Test uses: ./.github/actions/test - name: Build - run: CGO_ENABLED=1 go install github.com/mattn/go-sqlite3 && go build -o bin/ ./cmd/explored + run: | + go install github.com/mattn/go-sqlite3 + go build -o bin/ ./cmd/explored From b7751ad57a8097ca78e43cbf729ec950d4332269 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Thu, 4 Jan 2024 13:06:45 -0500 Subject: [PATCH 11/18] update workflow go versions --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 31a8c86f..2ea8ce6b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest , macos-latest, windows-latest ] - go-version: [ '1.19', '1.20' ] + go-version: [ '1.20', '1.21' ] steps: - name: Configure git run: git config --global core.autocrlf false # required on Windows From 6a2adabef89da83534acc37fb4ab60805db77767 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Fri, 5 Jan 2024 13:19:47 -0500 Subject: [PATCH 12/18] fixes for nate's review --- persist/sqlite/query.go | 1 + persist/sqlite/txn.go | 42 ++++++++++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/persist/sqlite/query.go b/persist/sqlite/query.go index a94d25eb..017d7d66 100644 --- a/persist/sqlite/query.go +++ b/persist/sqlite/query.go @@ -54,6 +54,7 @@ func (s *Store) Block(id types.BlockID) (result types.Block, err error) { if rows, err = s.query("SELECT address, value FROM MinerPayouts WHERE block_id = ? ORDER BY block_order", encode(id)); err != nil { return } + defer rows.Close() var address, value []byte for rows.Next() { diff --git a/persist/sqlite/txn.go b/persist/sqlite/txn.go index f091a6c8..0924890a 100644 --- a/persist/sqlite/txn.go +++ b/persist/sqlite/txn.go @@ -2,6 +2,7 @@ package sqlite import ( "bytes" + "fmt" "go.sia.tech/core/chain" "go.sia.tech/core/types" @@ -23,36 +24,47 @@ func encodeUint64(x uint64) []byte { return buf.Bytes() } -func (s *Store) addBlock(b types.Block, height uint64) error { +func (s *Store) addBlock(tx txn, b types.Block, height uint64) error { // nonce is encoded because database/sql doesn't support uint64 with high bit set - _, err := s.exec("INSERT INTO Blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encodeUint64(b.Nonce), b.Timestamp.Unix()) + _, err := tx.Exec("INSERT INTO Blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encodeUint64(b.Nonce), b.Timestamp.Unix()) return err } -func (s *Store) addMinerPayouts(bid types.BlockID, scos []types.SiacoinOutput) error { +func (s *Store) addMinerPayouts(tx txn, bid types.BlockID, scos []types.SiacoinOutput) error { for i, sco := range scos { - if _, err := s.exec("INSERT INTO MinerPayouts(block_id, block_order, address, value) VALUES (?, ?, ?, ?);", encode(bid), i, encode(sco.Address), encode(sco.Value)); err != nil { + if _, err := tx.Exec("INSERT INTO MinerPayouts(block_id, block_order, address, value) VALUES (?, ?, ?, ?);", encode(bid), i, encode(sco.Address), encode(sco.Value)); err != nil { return err } } return nil } -func (s *Store) deleteBlock(bid types.BlockID) error { - _, err := s.exec("DELETE FROM Blocks WHERE id = ?", encode(bid)) +func (s *Store) deleteBlock(tx txn, bid types.BlockID) error { + _, err := tx.Exec("DELETE FROM Blocks WHERE id = ?", encode(bid)) return err } func (s *Store) applyUpdates() error { - for _, update := range s.pendingUpdates { - if err := s.addBlock(update.Block, update.State.Index.Height); err != nil { - return err - } else if err := s.addMinerPayouts(update.Block.ID(), update.Block.MinerPayouts); err != nil { - return err + return s.transaction(func(tx txn) error { + for _, update := range s.pendingUpdates { + if err := s.addBlock(tx, update.Block, update.State.Index.Height); err != nil { + return fmt.Errorf("applyUpdates: failed to add block: %v", err) + } else if err := s.addMinerPayouts(tx, update.Block.ID(), update.Block.MinerPayouts); err != nil { + return fmt.Errorf("applyUpdates: failed to add miner payouts: %v", err) + } } - } - s.pendingUpdates = s.pendingUpdates[:0] - return nil + s.pendingUpdates = s.pendingUpdates[:0] + return nil + }) +} + +func (s *Store) revertUpdate(cru *chain.RevertUpdate) error { + return s.transaction(func(tx txn) error { + if err := s.deleteBlock(tx, cru.Block.ID()); err != nil { + return fmt.Errorf("revertUpdate: failed to delete block: %v", err) + } + return nil + }) } // ProcessChainApplyUpdate implements chain.Subscriber. @@ -75,5 +87,5 @@ func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { if err := s.applyUpdates(); err != nil { return err } - return s.deleteBlock(cru.Block.ID()) + return s.revertUpdate(cru) } From 6fe5da96d2e26eae151d65ae6caaa7f9a5b52eae Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Fri, 5 Jan 2024 14:11:29 -0500 Subject: [PATCH 13/18] return ErrNoTip if no blocks are present in Tip(), rename block methods to be more clear, and use snake_case SQL table names --- api/server.go | 8 ++++---- cmd/explored/node.go | 4 +++- explorer/explorer.go | 16 ++++++++-------- persist/sqlite/init.sql | 6 +++--- persist/sqlite/query.go | 31 ++++++++++++++++++++++--------- persist/sqlite/txn.go | 6 +++--- 6 files changed, 43 insertions(+), 28 deletions(-) diff --git a/api/server.go b/api/server.go index 6549338e..a1fd030b 100644 --- a/api/server.go +++ b/api/server.go @@ -40,8 +40,8 @@ type ( // Explorer implements a Sia explorer. Explorer interface { Tip() (types.ChainIndex, error) - Block(id types.BlockID) (types.Block, error) - BlockHeight(height uint64) (types.Block, error) + BlockByID(id types.BlockID) (types.Block, error) + BlockByHeight(height uint64) (types.Block, error) } ) @@ -145,7 +145,7 @@ func (s *server) explorerBlockHandler(jc jape.Context) { if jc.DecodeParam("id", &id) != nil { return } - block, err := s.e.Block(id) + block, err := s.e.BlockByID(id) if jc.Check("failed to get block", err) != nil { return } @@ -157,7 +157,7 @@ func (s *server) explorerBlockHeightHandler(jc jape.Context) { if jc.DecodeParam("height", &height) != nil { return } - block, err := s.e.BlockHeight(height) + block, err := s.e.BlockByHeight(height) if jc.Check("failed to get block", err) != nil { return } diff --git a/cmd/explored/node.go b/cmd/explored/node.go index c8f07e7b..f4278330 100644 --- a/cmd/explored/node.go +++ b/cmd/explored/node.go @@ -162,11 +162,13 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool, logger *zap.Lo } e := explorer.NewExplorer(store) tip, err := store.Tip() - if err != nil { + if errors.Is(err, sqlite.ErrNoTip) { tip = types.ChainIndex{ ID: genesisBlock.ID(), Height: 0, } + } else if err != nil { + return nil, err } cm.AddSubscriber(store, tip) diff --git a/explorer/explorer.go b/explorer/explorer.go index f45b7b2b..4865cd20 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -11,8 +11,8 @@ type Store interface { chain.Subscriber Tip() (types.ChainIndex, error) - Block(id types.BlockID) (types.Block, error) - BlockHeight(height uint64) (types.Block, error) + BlockByID(id types.BlockID) (types.Block, error) + BlockByHeight(height uint64) (types.Block, error) } // Explorer implements a Sia explorer. @@ -30,12 +30,12 @@ func (e *Explorer) Tip() (types.ChainIndex, error) { return e.s.Tip() } -// Block returns the block with the specified ID. -func (e *Explorer) Block(id types.BlockID) (types.Block, error) { - return e.s.Block(id) +// BlockByID returns the block with the specified ID. +func (e *Explorer) BlockByID(id types.BlockID) (types.Block, error) { + return e.s.BlockByID(id) } -// BlockHeight returns the block with the specified height. -func (e *Explorer) BlockHeight(height uint64) (types.Block, error) { - return e.s.BlockHeight(height) +// BlockByHeight returns the block with the specified height. +func (e *Explorer) BlockByHeight(height uint64) (types.Block, error) { + return e.s.BlockByHeight(height) } diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 97ff509d..c78eb8d7 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -3,7 +3,7 @@ CREATE TABLE global_settings ( db_version INTEGER NOT NULL -- used for migrations ); -CREATE TABLE Blocks ( +CREATE TABLE blocks ( id BINARY(32) NOT NULL PRIMARY KEY, height INTEGER NOT NULL, parent_id BINARY(32) NOT NULL, @@ -11,8 +11,8 @@ CREATE TABLE Blocks ( timestamp INTEGER NOT NULL ); -CREATE TABLE MinerPayouts ( - block_id REFERENCES Blocks(id) ON DELETE CASCADE, +CREATE TABLE miner_payouts ( + block_id REFERENCES blocks(id) ON DELETE CASCADE, block_order INTEGER NOT NULL, address BINARY(32) NOT NULL, value BINARY(16) NOT NULL diff --git a/persist/sqlite/query.go b/persist/sqlite/query.go index 017d7d66..91e7bc07 100644 --- a/persist/sqlite/query.go +++ b/persist/sqlite/query.go @@ -1,11 +1,20 @@ package sqlite import ( + "database/sql" + "errors" "time" "go.sia.tech/core/types" ) +var ( + // ErrNoTip is returned when Tip() is unable to find any blocks in the + // database and thus there is no tip. It does not mean there was an + // error in the underlying database. + ErrNoTip = errors.New("no tip found") +) + func decode(obj types.DecoderFrom, data []byte) error { d := types.NewBufDecoder(data) obj.DecodeFrom(d) @@ -23,7 +32,11 @@ func decodeUint64(x *uint64, data []byte) error { // Tip implements explorer.Store. func (s *Store) Tip() (result types.ChainIndex, err error) { var data []byte - if err = s.queryRow("SELECT id, height FROM Blocks WHERE height = (SELECT MAX(height) from Blocks)").Scan(&data, &result.Height); err != nil { + err = s.queryRow("SELECT id, height FROM blocks WHERE height = (SELECT MAX(height) from blocks)").Scan(&data, &result.Height) + if errors.Is(err, sql.ErrNoRows) { + err = ErrNoTip + return + } else if err != nil { return } if err = decode(&result.ID, data); err != nil { @@ -32,12 +45,12 @@ func (s *Store) Tip() (result types.ChainIndex, err error) { return } -// Block implements explorer.Store. -func (s *Store) Block(id types.BlockID) (result types.Block, err error) { +// BlockByID implements explorer.Store. +func (s *Store) BlockByID(id types.BlockID) (result types.Block, err error) { { var timestamp int64 var parentID, nonce []byte - if err = s.queryRow("SELECT parent_id, nonce, timestamp FROM Blocks WHERE id = ?", encode(id)).Scan(&parentID, &nonce, ×tamp); err != nil { + if err = s.queryRow("SELECT parent_id, nonce, timestamp FROM blocks WHERE id = ?", encode(id)).Scan(&parentID, &nonce, ×tamp); err != nil { return } result.Timestamp = time.Unix(timestamp, 0).UTC() @@ -51,7 +64,7 @@ func (s *Store) Block(id types.BlockID) (result types.Block, err error) { { var rows *loggedRows - if rows, err = s.query("SELECT address, value FROM MinerPayouts WHERE block_id = ? ORDER BY block_order", encode(id)); err != nil { + if rows, err = s.query("SELECT address, value FROM miner_payouts WHERE block_id = ? ORDER BY block_order", encode(id)); err != nil { return } defer rows.Close() @@ -75,10 +88,10 @@ func (s *Store) Block(id types.BlockID) (result types.Block, err error) { return } -// BlockHeight implements explorer.Store. -func (s *Store) BlockHeight(height uint64) (result types.Block, err error) { +// BlockByHeight implements explorer.Store. +func (s *Store) BlockByHeight(height uint64) (result types.Block, err error) { var data []byte - if err = s.queryRow("SELECT id FROM Blocks WHERE height = ?", height).Scan(&data); err != nil { + if err = s.queryRow("SELECT id FROM blocks WHERE height = ?", height).Scan(&data); err != nil { return } @@ -86,6 +99,6 @@ func (s *Store) BlockHeight(height uint64) (result types.Block, err error) { if err = decode(&bid, data); err != nil { return } - result, err = s.Block(bid) + result, err = s.BlockByID(bid) return } diff --git a/persist/sqlite/txn.go b/persist/sqlite/txn.go index 0924890a..490f7657 100644 --- a/persist/sqlite/txn.go +++ b/persist/sqlite/txn.go @@ -26,13 +26,13 @@ func encodeUint64(x uint64) []byte { func (s *Store) addBlock(tx txn, b types.Block, height uint64) error { // nonce is encoded because database/sql doesn't support uint64 with high bit set - _, err := tx.Exec("INSERT INTO Blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encodeUint64(b.Nonce), b.Timestamp.Unix()) + _, err := tx.Exec("INSERT INTO blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encodeUint64(b.Nonce), b.Timestamp.Unix()) return err } func (s *Store) addMinerPayouts(tx txn, bid types.BlockID, scos []types.SiacoinOutput) error { for i, sco := range scos { - if _, err := tx.Exec("INSERT INTO MinerPayouts(block_id, block_order, address, value) VALUES (?, ?, ?, ?);", encode(bid), i, encode(sco.Address), encode(sco.Value)); err != nil { + if _, err := tx.Exec("INSERT INTO miner_payouts(block_id, block_order, address, value) VALUES (?, ?, ?, ?);", encode(bid), i, encode(sco.Address), encode(sco.Value)); err != nil { return err } } @@ -40,7 +40,7 @@ func (s *Store) addMinerPayouts(tx txn, bid types.BlockID, scos []types.SiacoinO } func (s *Store) deleteBlock(tx txn, bid types.BlockID) error { - _, err := tx.Exec("DELETE FROM Blocks WHERE id = ?", encode(bid)) + _, err := tx.Exec("DELETE FROM blocks WHERE id = ?", encode(bid)) return err } From 93bede6d27a6a3888494c8aaf65901a7ce57f4a8 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Fri, 5 Jan 2024 14:22:25 -0500 Subject: [PATCH 14/18] remove unnecessary install from workflow --- .github/workflows/main.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2ea8ce6b..cabf5dfb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,6 +27,4 @@ jobs: - name: Test uses: ./.github/actions/test - name: Build - run: | - go install github.com/mattn/go-sqlite3 - go build -o bin/ ./cmd/explored + run: go build -o bin/ ./cmd/explored From ca33679e4965dbe3045a2823d6663c87f85fb675 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Fri, 5 Jan 2024 14:29:18 -0500 Subject: [PATCH 15/18] sql changes --- persist/sqlite/init.sql | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index c78eb8d7..0aa6c968 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -4,18 +4,19 @@ CREATE TABLE global_settings ( ); CREATE TABLE blocks ( - id BINARY(32) NOT NULL PRIMARY KEY, + id BLOB NOT NULL PRIMARY KEY, height INTEGER NOT NULL, - parent_id BINARY(32) NOT NULL, - nonce BINARY(8) NOT NULL, + parent_id BLOB NOT NULL, + nonce BLOB NOT NULL, timestamp INTEGER NOT NULL ); CREATE TABLE miner_payouts ( - block_id REFERENCES blocks(id) ON DELETE CASCADE, + block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL PRIMARY KEY, block_order INTEGER NOT NULL, - address BINARY(32) NOT NULL, - value BINARY(16) NOT NULL + address BLOB NOT NULL, + value BLOB NOT NULL, + UNIQUE(block_id, block_order) ); From 4f65e166ab7ce110e5cee4e1d20234e2cb2cd981 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Fri, 5 Jan 2024 14:47:14 -0500 Subject: [PATCH 16/18] create block_id index for miner_payouts --- persist/sqlite/init.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 0aa6c968..50043329 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -12,13 +12,14 @@ CREATE TABLE blocks ( ); CREATE TABLE miner_payouts ( - block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL PRIMARY KEY, + block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, block_order INTEGER NOT NULL, address BLOB NOT NULL, value BLOB NOT NULL, UNIQUE(block_id, block_order) ); +CREATE INDEX miner_payouts_index ON miner_payouts(block_id); -- initialize the global settings table INSERT INTO global_settings (id, db_version) VALUES (0, 0); -- should not be changed From 54f2b5dd204a9e3ab1138253ad016245183a3b20 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Mon, 8 Jan 2024 10:26:01 -0500 Subject: [PATCH 17/18] update revert --- persist/sqlite/txn.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/persist/sqlite/txn.go b/persist/sqlite/txn.go index 490f7657..ccb0a630 100644 --- a/persist/sqlite/txn.go +++ b/persist/sqlite/txn.go @@ -84,8 +84,9 @@ func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { s.mu.Lock() defer s.mu.Unlock() - if err := s.applyUpdates(); err != nil { - return err + if len(s.pendingUpdates) > 0 && s.pendingUpdates[len(s.pendingUpdates)-1].Block.ID() == cru.Block.ID() { + s.pendingUpdates = s.pendingUpdates[:len(s.pendingUpdates)-1] + return nil } return s.revertUpdate(cru) } From 7d21c167c1671b4ae3f9761a50d3a59c7b451ecb Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Mon, 8 Jan 2024 10:46:17 -0500 Subject: [PATCH 18/18] store database in correct directory --- cmd/explored/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/explored/node.go b/cmd/explored/node.go index f4278330..81ca9eb1 100644 --- a/cmd/explored/node.go +++ b/cmd/explored/node.go @@ -156,7 +156,7 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool, logger *zap.Lo } cm := chain.NewManager(dbstore, tipState) - store, err := sqlite.OpenDatabase("./explore.db", logger) + store, err := sqlite.OpenDatabase(filepath.Join(dir, "./explore.db"), logger) if err != nil { return nil, err }