From bcaef1bc6c413b27abad59a8cc6b41d4ac5735b3 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Thu, 18 Apr 2024 18:30:01 -0400 Subject: [PATCH 01/19] create updateTx struct and functions on it, rename loggedTxn -> txn, loggedStmt -> stmt, etc in sql.go --- explorer/explorer.go | 7 -- persist/sqlite/blocks.go | 4 +- persist/sqlite/consensus.go | 148 ++++++++++++++++----------------- persist/sqlite/contracts.go | 2 +- persist/sqlite/init.go | 4 +- persist/sqlite/merkle.go | 96 --------------------- persist/sqlite/migrations.go | 2 +- persist/sqlite/outputs.go | 6 +- persist/sqlite/sql.go | 125 ++++++++++++---------------- persist/sqlite/store.go | 6 +- persist/sqlite/transactions.go | 26 +++--- 11 files changed, 153 insertions(+), 273 deletions(-) delete mode 100644 persist/sqlite/merkle.go diff --git a/explorer/explorer.go b/explorer/explorer.go index 1160b1d2..a4a47bc6 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -39,8 +39,6 @@ type Store interface { UnspentSiafundOutputs(address types.Address, limit, offset uint64) ([]SiafundOutput, error) Balance(address types.Address) (sc types.Currency, immatureSC types.Currency, sf uint64, err error) Contracts(ids []types.FileContractID) (result []FileContract, err error) - - MerkleProof(leafIndex uint64) ([]types.Hash256, error) } // Explorer implements a Sia explorer. @@ -111,11 +109,6 @@ func NewExplorer(cm ChainManager, store Store, log *zap.Logger) (*Explorer, erro return e, nil } -// MerkleProof gets the merkle proof with the given leaf index. -func (e *Explorer) MerkleProof(leafIndex uint64) ([]types.Hash256, error) { - return e.s.MerkleProof(leafIndex) -} - // Tip returns the tip of the best known valid chain. func (e *Explorer) Tip() (types.ChainIndex, error) { return e.s.Tip() diff --git a/persist/sqlite/blocks.go b/persist/sqlite/blocks.go index a3171422..45c1338a 100644 --- a/persist/sqlite/blocks.go +++ b/persist/sqlite/blocks.go @@ -9,7 +9,7 @@ import ( // Block implements explorer.Store. func (s *Store) Block(id types.BlockID) (result explorer.Block, err error) { - err = s.transaction(func(tx txn) error { + err = s.transaction(func(tx *txn) error { err = tx.QueryRow(`SELECT parent_id, nonce, timestamp, height FROM blocks WHERE id=?`, dbEncode(id)).Scan(dbDecode(&result.ParentID), dbDecode(&result.Nonce), dbDecode(&result.Timestamp), &result.Height) if err != nil { return err @@ -38,7 +38,7 @@ func (s *Store) Block(id types.BlockID) (result explorer.Block, err error) { // BestTip implements explorer.Store. func (s *Store) BestTip(height uint64) (result types.ChainIndex, err error) { - err = s.transaction(func(tx txn) error { + err = s.transaction(func(tx *txn) error { err = tx.QueryRow(`SELECT id, height FROM blocks WHERE height=?`, height).Scan(dbDecode(&result.ID), dbDecode(&result.Height)) if err != nil { return err diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index bb74204a..65e89965 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -10,14 +10,19 @@ import ( "go.sia.tech/explored/explorer" ) -func (s *Store) addBlock(dbTxn txn, b types.Block, height uint64) error { +type updateTx struct { + tx *txn + relevantAddresses map[types.Address]bool +} + +func (ut *updateTx) AddBlock(b types.Block, height uint64) error { // nonce is encoded because database/sql doesn't support uint64 with high bit set - _, err := dbTxn.Exec("INSERT INTO blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", dbEncode(b.ID()), height, dbEncode(b.ParentID), dbEncode(b.Nonce), dbEncode(b.Timestamp)) + _, err := ut.tx.Exec("INSERT INTO blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", dbEncode(b.ID()), height, dbEncode(b.ParentID), dbEncode(b.Nonce), dbEncode(b.Timestamp)) return err } -func (s *Store) addMinerPayouts(dbTxn txn, bid types.BlockID, height uint64, scos []types.SiacoinOutput, dbIDs map[types.SiacoinOutputID]int64) error { - stmt, err := dbTxn.Prepare(`INSERT INTO miner_payouts(block_id, block_order, output_id) VALUES (?, ?, ?);`) +func (ut *updateTx) AddMinerPayouts(bid types.BlockID, height uint64, scos []types.SiacoinOutput, dbIDs map[types.SiacoinOutputID]int64) error { + stmt, err := ut.tx.Prepare(`INSERT INTO miner_payouts(block_id, block_order, output_id) VALUES (?, ?, ?);`) if err != nil { return fmt.Errorf("addMinerPayouts: failed to prepare statement: %w", err) } @@ -36,8 +41,9 @@ func (s *Store) addMinerPayouts(dbTxn txn, bid types.BlockID, height uint64, sco return nil } -func (s *Store) addArbitraryData(dbTxn txn, id int64, txn types.Transaction) error { - stmt, err := dbTxn.Prepare(`INSERT INTO transaction_arbitrary_data(transaction_id, transaction_order, data) VALUES (?, ?, ?)`) +func (ut *updateTx) AddArbitraryData(id int64, txn types.Transaction) error { + stmt, err := ut.tx.Prepare(`INSERT INTO transaction_arbitrary_data(transaction_id, transaction_order, data) VALUES (?, ?, ?)`) + if err != nil { return fmt.Errorf("addArbitraryData: failed to prepare statement: %w", err) } @@ -51,8 +57,8 @@ func (s *Store) addArbitraryData(dbTxn txn, id int64, txn types.Transaction) err return nil } -func (s *Store) addSiacoinInputs(dbTxn txn, id int64, txn types.Transaction) error { - stmt, err := dbTxn.Prepare(`INSERT INTO transaction_siacoin_inputs(transaction_id, transaction_order, parent_id, unlock_conditions) VALUES (?, ?, ?, ?)`) +func (ut *updateTx) AddSiacoinInputs(id int64, txn types.Transaction) error { + stmt, err := ut.tx.Prepare(`INSERT INTO transaction_siacoin_inputs(transaction_id, transaction_order, parent_id, unlock_conditions) VALUES (?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addSiacoinInputs: failed to prepare statement: %w", err) } @@ -66,8 +72,8 @@ func (s *Store) addSiacoinInputs(dbTxn txn, id int64, txn types.Transaction) err return nil } -func (s *Store) addSiacoinOutputs(dbTxn txn, id int64, txn types.Transaction, dbIDs map[types.SiacoinOutputID]int64) error { - stmt, err := dbTxn.Prepare(`INSERT INTO transaction_siacoin_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, ?)`) +func (ut *updateTx) AddSiacoinOutputs(id int64, txn types.Transaction, dbIDs map[types.SiacoinOutputID]int64) error { + stmt, err := ut.tx.Prepare(`INSERT INTO transaction_siacoin_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, ?)`) if err != nil { return fmt.Errorf("addSiacoinOutputs: failed to prepare statement: %w", err) } @@ -86,8 +92,8 @@ func (s *Store) addSiacoinOutputs(dbTxn txn, id int64, txn types.Transaction, db return nil } -func (s *Store) addSiafundInputs(dbTxn txn, id int64, txn types.Transaction) error { - stmt, err := dbTxn.Prepare(`INSERT INTO transaction_siafund_inputs(transaction_id, transaction_order, parent_id, unlock_conditions, claim_address) VALUES (?, ?, ?, ?, ?)`) +func (ut *updateTx) AddSiafundInputs(id int64, txn types.Transaction) error { + stmt, err := ut.tx.Prepare(`INSERT INTO transaction_siafund_inputs(transaction_id, transaction_order, parent_id, unlock_conditions, claim_address) VALUES (?, ?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addSiafundInputs: failed to prepare statement: %w", err) } @@ -101,8 +107,8 @@ func (s *Store) addSiafundInputs(dbTxn txn, id int64, txn types.Transaction) err return nil } -func (s *Store) addSiafundOutputs(dbTxn txn, id int64, txn types.Transaction, dbIDs map[types.SiafundOutputID]int64) error { - stmt, err := dbTxn.Prepare(`INSERT INTO transaction_siafund_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, ?)`) +func (ut *updateTx) AddSiafundOutputs(id int64, txn types.Transaction, dbIDs map[types.SiafundOutputID]int64) error { + stmt, err := ut.tx.Prepare(`INSERT INTO transaction_siafund_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, ?)`) if err != nil { return fmt.Errorf("addSiafundOutputs: failed to prepare statement: %w", err) } @@ -121,20 +127,20 @@ func (s *Store) addSiafundOutputs(dbTxn txn, id int64, txn types.Transaction, db return nil } -func (s *Store) addFileContracts(dbTxn txn, id int64, txn types.Transaction, fcDBIds map[fileContract]int64) error { - stmt, err := dbTxn.Prepare(`INSERT INTO transaction_file_contracts(transaction_id, transaction_order, contract_id) VALUES (?, ?, ?)`) +func (ut *updateTx) AddFileContracts(id int64, txn types.Transaction, fcDBIds map[fileContract]int64) error { + stmt, err := ut.tx.Prepare(`INSERT INTO transaction_file_contracts(transaction_id, transaction_order, contract_id) VALUES (?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContracts: failed to prepare statement: %w", err) } defer stmt.Close() - validOutputsStmt, err := dbTxn.Prepare(`INSERT INTO file_contract_valid_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) + validOutputsStmt, err := ut.tx.Prepare(`INSERT INTO file_contract_valid_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContracts: failed to prepare valid proof outputs statement: %w", err) } defer validOutputsStmt.Close() - missedOutputsStmt, err := dbTxn.Prepare(`INSERT INTO file_contract_missed_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) + missedOutputsStmt, err := ut.tx.Prepare(`INSERT INTO file_contract_missed_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContracts: failed to prepare missed proof outputs statement: %w", err) } @@ -165,20 +171,20 @@ func (s *Store) addFileContracts(dbTxn txn, id int64, txn types.Transaction, fcD return nil } -func (s *Store) addFileContractRevisions(dbTxn txn, id int64, txn types.Transaction, dbIDs map[fileContract]int64) error { - stmt, err := dbTxn.Prepare(`INSERT INTO transaction_file_contract_revisions(transaction_id, transaction_order, contract_id, parent_id, unlock_conditions) VALUES (?, ?, ?, ?, ?)`) +func (ut *updateTx) AddFileContractRevisions(id int64, txn types.Transaction, dbIDs map[fileContract]int64) error { + stmt, err := ut.tx.Prepare(`INSERT INTO transaction_file_contract_revisions(transaction_id, transaction_order, contract_id, parent_id, unlock_conditions) VALUES (?, ?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContractRevisions: failed to prepare statement: %w", err) } defer stmt.Close() - validOutputsStmt, err := dbTxn.Prepare(`INSERT INTO file_contract_valid_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) + validOutputsStmt, err := ut.tx.Prepare(`INSERT INTO file_contract_valid_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContracts: failed to prepare valid proof outputs statement: %w", err) } defer validOutputsStmt.Close() - missedOutputsStmt, err := dbTxn.Prepare(`INSERT INTO file_contract_missed_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) + missedOutputsStmt, err := ut.tx.Prepare(`INSERT INTO file_contract_missed_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContracts: failed to prepare missed proof outputs statement: %w", err) } @@ -211,8 +217,8 @@ func (s *Store) addFileContractRevisions(dbTxn txn, id int64, txn types.Transact return nil } -func (s *Store) addTransactions(dbTxn txn, bid types.BlockID, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[fileContract]int64) error { - insertTransactionStmt, err := dbTxn.Prepare(`INSERT INTO transactions (transaction_id) VALUES (?) +func (ut *updateTx) AddTransactions(bid types.BlockID, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[fileContract]int64) error { + insertTransactionStmt, err := ut.tx.Prepare(`INSERT INTO transactions (transaction_id) VALUES (?) ON CONFLICT (transaction_id) DO UPDATE SET transaction_id=EXCLUDED.transaction_id -- technically a no-op, but necessary for the RETURNING clause RETURNING id;`) if err != nil { @@ -220,7 +226,7 @@ func (s *Store) addTransactions(dbTxn txn, bid types.BlockID, txns []types.Trans } defer insertTransactionStmt.Close() - blockTransactionsStmt, err := dbTxn.Prepare(`INSERT INTO block_transactions(block_id, transaction_id, block_order) VALUES (?, ?, ?);`) + blockTransactionsStmt, err := ut.tx.Prepare(`INSERT INTO block_transactions(block_id, transaction_id, block_order) VALUES (?, ?, ?);`) if err != nil { return fmt.Errorf("failed to prepare block_transactions statement: %w", err) } @@ -235,19 +241,19 @@ func (s *Store) addTransactions(dbTxn txn, bid types.BlockID, txns []types.Trans if _, err := blockTransactionsStmt.Exec(dbEncode(bid), txnID, i); err != nil { return fmt.Errorf("failed to insert into block_transactions: %w", err) - } else if err := s.addArbitraryData(dbTxn, txnID, txn); err != nil { + } else if err := ut.AddArbitraryData(txnID, txn); err != nil { return fmt.Errorf("failed to add arbitrary data: %w", err) - } else if err := s.addSiacoinInputs(dbTxn, txnID, txn); err != nil { + } else if err := ut.AddSiacoinInputs(txnID, txn); err != nil { return fmt.Errorf("failed to add siacoin inputs: %w", err) - } else if err := s.addSiacoinOutputs(dbTxn, txnID, txn, scDBIds); err != nil { + } else if err := ut.AddSiacoinOutputs(txnID, txn, scDBIds); err != nil { return fmt.Errorf("failed to add siacoin outputs: %w", err) - } else if err := s.addSiafundInputs(dbTxn, txnID, txn); err != nil { + } else if err := ut.AddSiafundInputs(txnID, txn); err != nil { return fmt.Errorf("failed to add siafund inputs: %w", err) - } else if err := s.addSiafundOutputs(dbTxn, txnID, txn, sfDBIds); err != nil { + } else if err := ut.AddSiafundOutputs(txnID, txn, sfDBIds); err != nil { return fmt.Errorf("failed to add siafund outputs: %w", err) - } else if err := s.addFileContracts(dbTxn, txnID, txn, fcDBIds); err != nil { + } else if err := ut.AddFileContracts(txnID, txn, fcDBIds); err != nil { return fmt.Errorf("failed to add file contract: %w", err) - } else if err := s.addFileContractRevisions(dbTxn, txnID, txn, fcDBIds); err != nil { + } else if err := ut.AddFileContractRevisions(txnID, txn, fcDBIds); err != nil { return fmt.Errorf("failed to add file contract revisions: %w", err) } } @@ -266,7 +272,7 @@ type balance struct { sf uint64 } -func (s *Store) updateBalances(dbTxn txn, height uint64, spentSiacoinElements, newSiacoinElements []types.SiacoinElement, spentSiafundElements, newSiafundElements []types.SiafundElement) error { +func (ut *updateTx) UpdateBalances(height uint64, spentSiacoinElements, newSiacoinElements []types.SiacoinElement, spentSiafundElements, newSiafundElements []types.SiafundElement) error { addresses := make(map[types.Address]balance) for _, sce := range spentSiacoinElements { addresses[sce.SiacoinOutput.Address] = balance{} @@ -286,7 +292,7 @@ func (s *Store) updateBalances(dbTxn txn, height uint64, spentSiacoinElements, n addressList = append(addressList, dbEncode(address)) } - rows, err := dbTxn.Query(`SELECT address, siacoin_balance, immature_siacoin_balance, siafund_balance + rows, err := ut.tx.Query(`SELECT address, siacoin_balance, immature_siacoin_balance, siafund_balance FROM address_balance WHERE address IN (`+queryPlaceHolders(len(addressList))+`)`, addressList...) if err != nil { @@ -336,7 +342,7 @@ func (s *Store) updateBalances(dbTxn txn, height uint64, spentSiacoinElements, n addresses[sfe.SiafundOutput.Address] = bal } - stmt, err := dbTxn.Prepare(`INSERT INTO address_balance(address, siacoin_balance, immature_siacoin_balance, siafund_balance) + stmt, err := ut.tx.Prepare(`INSERT INTO address_balance(address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES (?, ?, ?, ?) ON CONFLICT(address) DO UPDATE set siacoin_balance = ?, immature_siacoin_balance = ?, siafund_balance = ?`) @@ -355,14 +361,14 @@ func (s *Store) updateBalances(dbTxn txn, height uint64, spentSiacoinElements, n return nil } -func (s *Store) updateMaturedBalances(dbTxn txn, revert bool, height uint64) error { +func (ut *updateTx) UpdateMaturedBalances(revert bool, height uint64) error { // Prevent double counting - outputs with a maturity height of 0 are // handled in updateBalances if height == 0 { return nil } - rows, err := dbTxn.Query(`SELECT address, value + rows, err := ut.tx.Query(`SELECT address, value FROM siacoin_elements WHERE maturity_height = ?`, height) if err != nil { @@ -381,7 +387,7 @@ func (s *Store) updateMaturedBalances(dbTxn txn, revert bool, height uint64) err addressList = append(addressList, dbEncode(sco.Address)) } - balanceRows, err := dbTxn.Query(`SELECT address, siacoin_balance, immature_siacoin_balance + balanceRows, err := ut.tx.Query(`SELECT address, siacoin_balance, immature_siacoin_balance FROM address_balance WHERE address IN (`+queryPlaceHolders(len(addressList))+`)`, addressList...) if err != nil { @@ -413,7 +419,7 @@ func (s *Store) updateMaturedBalances(dbTxn txn, revert bool, height uint64) err addresses[sco.Address] = bal } - stmt, err := dbTxn.Prepare(`INSERT INTO address_balance(address, siacoin_balance, immature_siacoin_balance, siafund_balance) + stmt, err := ut.tx.Prepare(`INSERT INTO address_balance(address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES (?, ?, ?, ?) ON CONFLICT(address) DO UPDATE set siacoin_balance = ?, immature_siacoin_balance = ?`) @@ -432,7 +438,7 @@ func (s *Store) updateMaturedBalances(dbTxn txn, revert bool, height uint64) err return nil } -func (s *Store) addSiacoinElements(dbTxn txn, bid types.BlockID, update consensusUpdate, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) { +func (ut *updateTx) AddSiacoinElements(bid types.BlockID, update consensusUpdate, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) { sources := make(map[types.SiacoinOutputID]explorer.Source) if applyUpdate, ok := update.(chain.ApplyUpdate); ok { block := applyUpdate.Block @@ -457,7 +463,7 @@ func (s *Store) addSiacoinElements(dbTxn txn, bid types.BlockID, update consensu } } - stmt, err := dbTxn.Prepare(`INSERT INTO siacoin_elements(output_id, block_id, leaf_index, merkle_proof, spent, source, maturity_height, address, value) + stmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements(output_id, block_id, leaf_index, merkle_proof, spent, source, maturity_height, address, value) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (output_id) DO UPDATE SET spent = ?`) @@ -497,8 +503,8 @@ func (s *Store) addSiacoinElements(dbTxn txn, bid types.BlockID, update consensu return scDBIds, nil } -func (s *Store) addSiafundElements(dbTxn txn, bid types.BlockID, spentElements, newElements []types.SiafundElement) (map[types.SiafundOutputID]int64, error) { - stmt, err := dbTxn.Prepare(`INSERT INTO siafund_elements(output_id, block_id, leaf_index, merkle_proof, spent, claim_start, address, value) +func (ut *updateTx) AddSiafundElements(bid types.BlockID, spentElements, newElements []types.SiafundElement) (map[types.SiafundOutputID]int64, error) { + stmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements(output_id, block_id, leaf_index, merkle_proof, spent, claim_start, address, value) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO UPDATE SET spent = ?`) @@ -543,8 +549,8 @@ type fileContract struct { revisionNumber uint64 } -func (s *Store) addFileContractElements(dbTxn txn, bid types.BlockID, update consensusUpdate) (map[fileContract]int64, error) { - stmt, err := dbTxn.Prepare(`INSERT INTO file_contract_elements(block_id, contract_id, leaf_index, merkle_proof, resolved, valid, filesize, file_merkle_root, window_start, window_end, payout, unlock_hash, revision_number) +func (ut *updateTx) AddFileContractElements(bid types.BlockID, update consensusUpdate) (map[fileContract]int64, error) { + stmt, err := ut.tx.Prepare(`INSERT INTO file_contract_elements(block_id, contract_id, leaf_index, merkle_proof, resolved, valid, filesize, file_merkle_root, window_start, window_end, payout, unlock_hash, revision_number) VALUES (?, ?, ?, ?, FALSE, TRUE, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (contract_id, revision_number) DO UPDATE SET resolved = ?, valid = ? @@ -554,7 +560,7 @@ func (s *Store) addFileContractElements(dbTxn txn, bid types.BlockID, update con } defer stmt.Close() - revisionStmt, err := dbTxn.Prepare(`INSERT INTO last_contract_revision(contract_id, contract_element_id) + revisionStmt, err := ut.tx.Prepare(`INSERT INTO last_contract_revision(contract_id, contract_element_id) VALUES (?, ?) ON CONFLICT (contract_id) DO UPDATE SET contract_element_id = ?`) @@ -591,16 +597,18 @@ func (s *Store) addFileContractElements(dbTxn txn, bid types.BlockID, update con return fcDBIds, updateErr } -func (s *Store) deleteBlock(dbTxn txn, bid types.BlockID) error { - _, err := dbTxn.Exec("DELETE FROM blocks WHERE id = ?", dbEncode(bid)) +func (ut *updateTx) DeleteBlock(bid types.BlockID) error { + _, err := ut.tx.Exec("DELETE FROM blocks WHERE id = ?", dbEncode(bid)) return err } // ProcessChainUpdates implements explorer.Store. func (s *Store) ProcessChainUpdates(crus []chain.RevertUpdate, caus []chain.ApplyUpdate) error { - return s.transaction(func(dbTxn txn) error { + return s.transaction(func(dbTxn *txn) error { + tx := &updateTx{tx: dbTxn} + for _, cru := range crus { - if err := s.updateMaturedBalances(dbTxn, true, cru.State.Index.Height+1); err != nil { + if err := tx.UpdateMaturedBalances(true, cru.State.Index.Height+1); err != nil { return fmt.Errorf("revertUpdate: failed to update matured balances: %w", err) } @@ -653,36 +661,32 @@ func (s *Store) ProcessChainUpdates(crus []chain.RevertUpdate, caus []chain.Appl }) // log.Println("REVERT!") - if _, err := s.addSiacoinElements( - dbTxn, + if _, err := tx.AddSiacoinElements( cru.Block.ID(), cru, spentSiacoinElements, append(newSiacoinElements, ephemeralSiacoinElements...), ); err != nil { return fmt.Errorf("revertUpdate: failed to update siacoin output state: %w", err) - } else if _, err := s.addSiafundElements( - dbTxn, + } else if _, err := tx.AddSiafundElements( cru.Block.ID(), spentSiafundElements, append(newSiafundElements, ephemeralSiafundElements...), ); err != nil { return fmt.Errorf("revertUpdate: failed to update siafund output state: %w", err) - } else if err := s.updateBalances(dbTxn, cru.State.Index.Height+1, spentSiacoinElements, newSiacoinElements, spentSiafundElements, newSiafundElements); err != nil { + } else if err := tx.UpdateBalances(cru.State.Index.Height+1, spentSiacoinElements, newSiacoinElements, spentSiafundElements, newSiafundElements); err != nil { return fmt.Errorf("revertUpdate: failed to update balances: %w", err) - } else if _, err := s.addFileContractElements(dbTxn, cru.Block.ID(), cru); err != nil { + } else if _, err := tx.AddFileContractElements(cru.Block.ID(), cru); err != nil { return fmt.Errorf("revertUpdate: failed to update file contract state: %w", err) - } else if err := s.updateLeaves(dbTxn, cru); err != nil { - return fmt.Errorf("revertUpdate: failed to update leaves: %w", err) - } else if err := s.deleteBlock(dbTxn, cru.Block.ID()); err != nil { + } else if err := tx.DeleteBlock(cru.Block.ID()); err != nil { return fmt.Errorf("revertUpdate: failed to delete block: %w", err) } } for _, cau := range caus { - if err := s.addBlock(dbTxn, cau.Block, cau.State.Index.Height); err != nil { + if err := tx.AddBlock(cau.Block, cau.State.Index.Height); err != nil { return fmt.Errorf("applyUpdates: failed to add block: %w", err) - } else if err := s.updateMaturedBalances(dbTxn, false, cau.State.Index.Height); err != nil { + } else if err := tx.UpdateMaturedBalances(false, cau.State.Index.Height); err != nil { return fmt.Errorf("applyUpdates: failed to update matured balances: %w", err) } @@ -734,8 +738,7 @@ func (s *Store) ProcessChainUpdates(crus []chain.RevertUpdate, caus []chain.Appl } }) - scDBIds, err := s.addSiacoinElements( - dbTxn, + scDBIds, err := tx.AddSiacoinElements( cau.Block.ID(), cau, append(spentSiacoinElements, ephemeralSiacoinElements...), @@ -744,8 +747,7 @@ func (s *Store) ProcessChainUpdates(crus []chain.RevertUpdate, caus []chain.Appl if err != nil { return fmt.Errorf("applyUpdates: failed to add siacoin outputs: %w", err) } - sfDBIds, err := s.addSiafundElements( - dbTxn, + sfDBIds, err := tx.AddSiafundElements( cau.Block.ID(), append(spentSiafundElements, ephemeralSiafundElements...), newSiafundElements, @@ -753,24 +755,20 @@ func (s *Store) ProcessChainUpdates(crus []chain.RevertUpdate, caus []chain.Appl if err != nil { return fmt.Errorf("applyUpdates: failed to add siafund outputs: %w", err) } - if err := s.updateBalances(dbTxn, cau.State.Index.Height, spentSiacoinElements, newSiacoinElements, spentSiafundElements, newSiafundElements); err != nil { + if err := tx.UpdateBalances(cau.State.Index.Height, spentSiacoinElements, newSiacoinElements, spentSiafundElements, newSiafundElements); err != nil { return fmt.Errorf("applyUpdates: failed to update balances: %w", err) } - fcDBIds, err := s.addFileContractElements(dbTxn, cau.Block.ID(), cau) + fcDBIds, err := tx.AddFileContractElements(cau.Block.ID(), cau) if err != nil { return fmt.Errorf("applyUpdates: failed to add file contracts: %w", err) } - if err := s.addMinerPayouts(dbTxn, cau.Block.ID(), cau.State.Index.Height, cau.Block.MinerPayouts, scDBIds); err != nil { + if err := tx.AddMinerPayouts(cau.Block.ID(), cau.State.Index.Height, cau.Block.MinerPayouts, scDBIds); err != nil { return fmt.Errorf("applyUpdates: failed to add miner payouts: %w", err) - } else if err := s.addTransactions(dbTxn, cau.Block.ID(), cau.Block.Transactions, scDBIds, sfDBIds, fcDBIds); err != nil { + } else if err := tx.AddTransactions(cau.Block.ID(), cau.Block.Transactions, scDBIds, sfDBIds, fcDBIds); err != nil { return fmt.Errorf("applyUpdates: failed to add transactions: addTransactions: %w", err) } - - if err := s.updateLeaves(dbTxn, cau); err != nil { - return err - } } return nil }) @@ -779,8 +777,8 @@ func (s *Store) ProcessChainUpdates(crus []chain.RevertUpdate, caus []chain.Appl // Tip implements explorer.Store. func (s *Store) Tip() (result types.ChainIndex, err error) { const query = `SELECT id, height FROM blocks ORDER BY height DESC LIMIT 1` - err = s.transaction(func(dbTx txn) error { - return dbTx.QueryRow(query).Scan(dbDecode(&result.ID), &result.Height) + err = s.transaction(func(dbTxn *txn) error { + return dbTxn.QueryRow(query).Scan(dbDecode(&result.ID), &result.Height) }) if errors.Is(err, sql.ErrNoRows) { return types.ChainIndex{}, explorer.ErrNoTip diff --git a/persist/sqlite/contracts.go b/persist/sqlite/contracts.go index ca85b8c8..4a5394ba 100644 --- a/persist/sqlite/contracts.go +++ b/persist/sqlite/contracts.go @@ -17,7 +17,7 @@ func (s *Store) Contracts(ids []types.FileContractID) (result []explorer.FileCon return result } - err = s.transaction(func(tx txn) error { + err = s.transaction(func(tx *txn) error { query := `SELECT fc1.id, fc1.contract_id, fc1.leaf_index, fc1.merkle_proof, fc1.resolved, fc1.valid, fc1.filesize, fc1.file_merkle_root, fc1.window_start, fc1.window_end, fc1.payout, fc1.unlock_hash, fc1.revision_number FROM file_contract_elements fc1 INNER JOIN last_contract_revision rev ON (rev.contract_element_id = fc1.id) diff --git a/persist/sqlite/init.go b/persist/sqlite/init.go index cda2ddb9..dbf4ee3c 100644 --- a/persist/sqlite/init.go +++ b/persist/sqlite/init.go @@ -17,7 +17,7 @@ import ( var initDatabase string func (s *Store) initNewDatabase(target int64) error { - return s.transaction(func(tx txn) error { + return s.transaction(func(tx *txn) error { if _, err := tx.Exec(initDatabase); err != nil { return fmt.Errorf("failed to initialize database: %w", err) } else if err := setDBVersion(tx, target); err != nil { @@ -42,7 +42,7 @@ func (s *Store) upgradeDatabase(current, target int64) error { } }() - return s.transaction(func(tx txn) error { + return s.transaction(func(tx *txn) error { for _, fn := range migrations[current-1:] { current++ start := time.Now() diff --git a/persist/sqlite/merkle.go b/persist/sqlite/merkle.go deleted file mode 100644 index a6c40dd1..00000000 --- a/persist/sqlite/merkle.go +++ /dev/null @@ -1,96 +0,0 @@ -package sqlite - -import ( - "math/bits" - - "go.sia.tech/core/types" -) - -func (s *Store) updateLeaves(dbTxn txn, update consensusUpdate) error { - modifyLeaf := func(stmt *loggedStmt, elem types.StateElement) error { - pos := elem.LeafIndex - for i, h := range elem.MerkleProof { - subtreeSize := uint64(1 << i) - if elem.LeafIndex&(1< s.numLeaves { - s.numLeaves = elem.LeafIndex + 1 - } - return nil - } - - stmt, err := dbTxn.Prepare(`INSERT INTO merkle_proofs(i, j, hash) VALUES (?, ?, ?) ON CONFLICT (i, j) DO UPDATE SET hash = ?;`) - if err != nil { - return err - } - - update.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { - if err != nil { - return - } - err = modifyLeaf(stmt, sce.StateElement) - return - }) - if err != nil { - return err - } - - update.ForEachSiafundElement(func(sce types.SiafundElement, spent bool) { - if err != nil { - return - } - err = modifyLeaf(stmt, sce.StateElement) - return - }) - if err != nil { - return err - } - - update.ForEachFileContractElement(func(sce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { - if err != nil { - return - } - err = modifyLeaf(stmt, sce.StateElement) - return - }) - if err != nil { - return err - } - - return nil -} - -// MerkleProof implements explorer.Store. -func (s *Store) MerkleProof(leafIndex uint64) ([]types.Hash256, error) { - proof := make([]types.Hash256, bits.Len64(leafIndex^s.numLeaves)-1) - err := s.transaction(func(tx txn) error { - pos := leafIndex - stmt, err := tx.Prepare("SELECT hash FROM merkle_proofs WHERE i = ? AND j = ?") - if err != nil { - return err - } - for i := range proof { - subtreeSize := uint64(1 << i) - if leafIndex&(1< longQueryDuration { - lr.log.Debug("slow next", zap.Duration("elapsed", dur), zap.Stack("stack")) + r.log.Debug("slow next", zap.Duration("elapsed", dur), zap.Stack("stack")) } return next } -func (lr *loggedRows) Scan(dest ...any) error { +func (r *rows) Scan(dest ...any) error { start := time.Now() - err := lr.Rows.Scan(dest...) + err := r.Rows.Scan(dest...) if dur := time.Since(start); dur > longQueryDuration { - lr.log.Debug("slow scan", zap.Duration("elapsed", dur), zap.Stack("stack")) + r.log.Debug("slow scan", zap.Duration("elapsed", dur), zap.Stack("stack")) } return err } -func (lr *loggedRow) Scan(dest ...any) error { +func (r *row) Scan(dest ...any) error { start := time.Now() - err := lr.Row.Scan(dest...) + err := r.Row.Scan(dest...) if dur := time.Since(start); dur > longQueryDuration { - lr.log.Debug("slow scan", zap.Duration("elapsed", dur), zap.Stack("stack")) + r.log.Debug("slow scan", zap.Duration("elapsed", dur), zap.Stack("stack")) } return err } -func (ls *loggedStmt) Exec(args ...any) (sql.Result, error) { - return ls.ExecContext(context.Background(), args...) +func (s *stmt) Exec(args ...any) (sql.Result, error) { + return s.ExecContext(context.Background(), args...) } -func (ls *loggedStmt) ExecContext(ctx context.Context, args ...any) (sql.Result, error) { +func (s *stmt) ExecContext(ctx context.Context, args ...any) (sql.Result, error) { start := time.Now() - result, err := ls.Stmt.ExecContext(ctx, args...) + result, err := s.Stmt.ExecContext(ctx, args...) if dur := time.Since(start); dur > longQueryDuration { - ls.log.Debug("slow exec", zap.String("query", ls.query), zap.Duration("elapsed", dur), zap.Stack("stack")) + s.log.Debug("slow exec", zap.String("query", s.query), zap.Duration("elapsed", dur), zap.Stack("stack")) } return result, err } -func (ls *loggedStmt) Query(args ...any) (*sql.Rows, error) { - return ls.QueryContext(context.Background(), args...) +func (s *stmt) Query(args ...any) (*sql.Rows, error) { + return s.QueryContext(context.Background(), args...) } -func (ls *loggedStmt) QueryContext(ctx context.Context, args ...any) (*sql.Rows, error) { +func (s *stmt) QueryContext(ctx context.Context, args ...any) (*sql.Rows, error) { start := time.Now() - rows, err := ls.Stmt.QueryContext(ctx, args...) + rows, err := s.Stmt.QueryContext(ctx, args...) if dur := time.Since(start); dur > longQueryDuration { - ls.log.Debug("slow query", zap.String("query", ls.query), zap.Duration("elapsed", dur), zap.Stack("stack")) + s.log.Debug("slow query", zap.String("query", s.query), zap.Duration("elapsed", dur), zap.Stack("stack")) } return rows, err } -func (ls *loggedStmt) QueryRow(args ...any) *loggedRow { - return ls.QueryRowContext(context.Background(), args...) +func (s *stmt) QueryRow(args ...any) *row { + return s.QueryRowContext(context.Background(), args...) } -func (ls *loggedStmt) QueryRowContext(ctx context.Context, args ...any) *loggedRow { +func (s *stmt) QueryRowContext(ctx context.Context, args ...any) *row { start := time.Now() - row := ls.Stmt.QueryRowContext(ctx, args...) + r := s.Stmt.QueryRowContext(ctx, args...) if dur := time.Since(start); dur > longQueryDuration { - ls.log.Debug("slow query row", zap.String("query", ls.query), zap.Duration("elapsed", dur), zap.Stack("stack")) + s.log.Debug("slow query row", zap.String("query", s.query), zap.Duration("elapsed", dur), zap.Stack("stack")) } - return &loggedRow{row, ls.log.Named("row")} + return &row{r, s.log.Named("row")} } // Exec executes a query without returning any rows. The args are for // any placeholder parameters in the query. -func (lt *loggedTxn) Exec(query string, args ...any) (sql.Result, error) { +func (tx *txn) Exec(query string, args ...any) (sql.Result, error) { start := time.Now() - result, err := lt.Tx.Exec(query, args...) + result, err := tx.Tx.Exec(query, args...) if dur := time.Since(start); dur > longQueryDuration { - lt.log.Debug("slow exec", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) + tx.log.Debug("slow exec", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) } return result, err } @@ -146,30 +131,30 @@ func (lt *loggedTxn) Exec(query string, args ...any) (sql.Result, error) { // Multiple queries or executions may be run concurrently from the // returned statement. The caller must call the statement's Close method // when the statement is no longer needed. -func (lt *loggedTxn) Prepare(query string) (*loggedStmt, error) { +func (tx *txn) Prepare(query string) (*stmt, error) { start := time.Now() - stmt, err := lt.Tx.Prepare(query) + s, err := tx.Tx.Prepare(query) if dur := time.Since(start); dur > longQueryDuration { - lt.log.Debug("slow prepare", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) + tx.log.Debug("slow prepare", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) } else if err != nil { return nil, err } - return &loggedStmt{ - Stmt: stmt, + return &stmt{ + Stmt: s, query: query, - log: lt.log.Named("statement"), + log: tx.log.Named("statement"), }, nil } // Query executes a query that returns rows, typically a SELECT. The // args are for any placeholder parameters in the query. -func (lt *loggedTxn) Query(query string, args ...any) (*loggedRows, error) { +func (tx *txn) Query(query string, args ...any) (*rows, error) { start := time.Now() - rows, err := lt.Tx.Query(query, args...) + r, err := tx.Tx.Query(query, args...) if dur := time.Since(start); dur > longQueryDuration { - lt.log.Debug("slow query", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) + tx.log.Debug("slow query", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) } - return &loggedRows{rows, lt.log.Named("rows")}, err + return &rows{r, tx.log.Named("rows")}, err } // QueryRow executes a query that is expected to return at most one row. @@ -177,13 +162,13 @@ func (lt *loggedTxn) Query(query string, args ...any) (*loggedRows, error) { // Row's Scan method is called. If the query selects no rows, the *Row's // Scan will return ErrNoRows. Otherwise, the *Row's Scan scans the // first selected row and discards the rest. -func (lt *loggedTxn) QueryRow(query string, args ...any) *loggedRow { +func (tx *txn) QueryRow(query string, args ...any) *row { start := time.Now() - row := lt.Tx.QueryRow(query, args...) + r := tx.Tx.QueryRow(query, args...) if dur := time.Since(start); dur > longQueryDuration { - lt.log.Debug("slow query row", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) + tx.log.Debug("slow query row", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) } - return &loggedRow{row, lt.log.Named("row")} + return &row{r, tx.log.Named("row")} } func queryPlaceHolders(n int) string { @@ -220,7 +205,7 @@ func getDBVersion(db *sql.DB) (version int64) { } // setDBVersion sets the current version of the database. -func setDBVersion(tx txn, version int64) error { +func setDBVersion(tx *txn, version int64) error { const query = `UPDATE global_settings SET db_version=$1 RETURNING id;` var dbID int64 return tx.QueryRow(query, version).Scan(&dbID) diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index c0ad275c..c0d29512 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -28,7 +28,7 @@ type ( // function returns an error, the transaction is rolled back. Otherwise, the // transaction is committed. If the transaction fails due to a busy error, it is // retried up to 10 times before returning. -func (s *Store) transaction(fn func(txn) error) error { +func (s *Store) transaction(fn func(*txn) error) error { var err error txnID := hex.EncodeToString(frand.Bytes(4)) log := s.log.Named("transaction").With(zap.String("id", txnID)) @@ -72,7 +72,7 @@ func sqliteFilepath(fp string) string { // doTransaction is a helper function to execute a function within a transaction. If fn returns // an error, the transaction is rolled back. Otherwise, the transaction is // committed. -func doTransaction(db *sql.DB, log *zap.Logger, fn func(tx txn) error) error { +func doTransaction(db *sql.DB, log *zap.Logger, fn func(tx *txn) error) error { start := time.Now() tx, err := db.Begin() if err != nil { @@ -86,7 +86,7 @@ func doTransaction(db *sql.DB, log *zap.Logger, fn func(tx txn) error) error { } }() - ltx := &loggedTxn{ + ltx := &txn{ Tx: tx, log: log, } diff --git a/persist/sqlite/transactions.go b/persist/sqlite/transactions.go index 17a6b698..57e33614 100644 --- a/persist/sqlite/transactions.go +++ b/persist/sqlite/transactions.go @@ -8,7 +8,7 @@ import ( ) // transactionArbitraryData returns the arbitrary data for each transaction. -func transactionArbitraryData(tx txn, txnIDs []int64) (map[int64][][]byte, error) { +func transactionArbitraryData(tx *txn, txnIDs []int64) (map[int64][][]byte, error) { query := `SELECT transaction_id, data FROM transaction_arbitrary_data WHERE transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `) @@ -32,7 +32,7 @@ ORDER BY transaction_order ASC` } // transactionSiacoinOutputs returns the siacoin outputs for each transaction. -func transactionSiacoinOutputs(tx txn, txnIDs []int64) (map[int64][]explorer.SiacoinOutput, error) { +func transactionSiacoinOutputs(tx *txn, txnIDs []int64) (map[int64][]explorer.SiacoinOutput, error) { query := `SELECT ts.transaction_id, sc.output_id, sc.leaf_index, sc.merkle_proof, sc.source, sc.maturity_height, sc.address, sc.value FROM siacoin_elements sc INNER JOIN transaction_siacoin_outputs ts ON (ts.output_id = sc.id) @@ -58,7 +58,7 @@ ORDER BY ts.transaction_order ASC` } // transactionSiacoinInputs returns the siacoin inputs for each transaction. -func transactionSiacoinInputs(tx txn, txnIDs []int64) (map[int64][]types.SiacoinInput, error) { +func transactionSiacoinInputs(tx *txn, txnIDs []int64) (map[int64][]types.SiacoinInput, error) { query := `SELECT transaction_id, parent_id, unlock_conditions FROM transaction_siacoin_inputs WHERE transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `) @@ -82,7 +82,7 @@ ORDER BY transaction_order ASC` } // transactionSiafundInputs returns the siafund inputs for each transaction. -func transactionSiafundInputs(tx txn, txnIDs []int64) (map[int64][]types.SiafundInput, error) { +func transactionSiafundInputs(tx *txn, txnIDs []int64) (map[int64][]types.SiafundInput, error) { query := `SELECT transaction_id, parent_id, unlock_conditions, claim_address FROM transaction_siafund_inputs WHERE transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `) @@ -106,7 +106,7 @@ ORDER BY transaction_order ASC` } // transactionSiafundOutputs returns the siafund outputs for each transaction. -func transactionSiafundOutputs(tx txn, txnIDs []int64) (map[int64][]explorer.SiafundOutput, error) { +func transactionSiafundOutputs(tx *txn, txnIDs []int64) (map[int64][]explorer.SiafundOutput, error) { query := `SELECT ts.transaction_id, sf.output_id, sf.leaf_index, sf.merkle_proof, sf.claim_start, sf.address, sf.value FROM siafund_elements sf INNER JOIN transaction_siafund_outputs ts ON (ts.output_id = sf.id) @@ -136,7 +136,7 @@ type fileContractProofOutputs struct { missed []types.SiacoinOutput } -func fileContractOutputs(tx txn, contractIDs []int64) (map[int64]fileContractProofOutputs, error) { +func fileContractOutputs(tx *txn, contractIDs []int64) (map[int64]fileContractProofOutputs, error) { result := make(map[int64]fileContractProofOutputs) validQuery := `SELECT contract_id, address, value @@ -192,7 +192,7 @@ type contractOrder struct { } // transactionFileContracts returns the file contracts for each transaction. -func transactionFileContracts(tx txn, txnIDs []int64) (map[int64][]explorer.FileContract, error) { +func transactionFileContracts(tx *txn, txnIDs []int64) (map[int64][]explorer.FileContract, error) { query := `SELECT ts.transaction_id, fc.id, fc.contract_id, fc.leaf_index, fc.merkle_proof, fc.resolved, fc.valid, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number FROM file_contract_elements fc INNER JOIN transaction_file_contracts ts ON (ts.contract_id = fc.id) @@ -235,7 +235,7 @@ ORDER BY ts.transaction_order ASC` } // transactionFileContracts returns the file contract revisions for each transaction. -func transactionFileContractRevisions(tx txn, txnIDs []int64) (map[int64][]explorer.FileContractRevision, error) { +func transactionFileContractRevisions(tx *txn, txnIDs []int64) (map[int64][]explorer.FileContractRevision, error) { query := `SELECT ts.transaction_id, fc.id, ts.parent_id, ts.unlock_conditions, fc.contract_id, fc.leaf_index, fc.merkle_proof, fc.resolved, fc.valid, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number FROM file_contract_elements fc INNER JOIN transaction_file_contract_revisions ts ON (ts.contract_id = fc.id) @@ -279,7 +279,7 @@ ORDER BY ts.transaction_order ASC` // blockTransactionIDs returns the database ID for each transaction in the // block. -func blockTransactionIDs(tx txn, blockID types.BlockID) (dbIDs []int64, err error) { +func blockTransactionIDs(tx *txn, blockID types.BlockID) (dbIDs []int64, err error) { rows, err := tx.Query(`SELECT transaction_id FROM block_transactions WHERE block_id = ? ORDER BY block_order`, dbEncode(blockID)) if err != nil { return nil, err @@ -297,7 +297,7 @@ func blockTransactionIDs(tx txn, blockID types.BlockID) (dbIDs []int64, err erro } // blockMinerPayouts returns the miner payouts for the block. -func blockMinerPayouts(tx txn, blockID types.BlockID) ([]explorer.SiacoinOutput, error) { +func blockMinerPayouts(tx *txn, blockID types.BlockID) ([]explorer.SiacoinOutput, error) { query := `SELECT sc.output_id, sc.leaf_index, sc.merkle_proof, sc.source, sc.maturity_height, sc.address, sc.value FROM siacoin_elements sc INNER JOIN miner_payouts mp ON (mp.output_id = sc.id) @@ -321,7 +321,7 @@ ORDER BY mp.block_order ASC` } // transactionDatabaseIDs returns the database ID for each transaction. -func transactionDatabaseIDs(tx txn, txnIDs []types.TransactionID) (dbIDs []int64, err error) { +func transactionDatabaseIDs(tx *txn, txnIDs []types.TransactionID) (dbIDs []int64, err error) { encodedIDs := func(ids []types.TransactionID) []any { result := make([]any, len(ids)) for i, id := range ids { @@ -347,7 +347,7 @@ func transactionDatabaseIDs(tx txn, txnIDs []types.TransactionID) (dbIDs []int64 return } -func (s *Store) getTransactions(tx txn, dbIDs []int64) ([]explorer.Transaction, error) { +func (s *Store) getTransactions(tx *txn, dbIDs []int64) ([]explorer.Transaction, error) { txnArbitraryData, err := transactionArbitraryData(tx, dbIDs) if err != nil { return nil, fmt.Errorf("getTransactions: failed to get arbitrary data: %w", err) @@ -404,7 +404,7 @@ func (s *Store) getTransactions(tx txn, dbIDs []int64) ([]explorer.Transaction, // Transactions implements explorer.Store. func (s *Store) Transactions(ids []types.TransactionID) (results []explorer.Transaction, err error) { - err = s.transaction(func(tx txn) error { + err = s.transaction(func(tx *txn) error { dbIDs, err := transactionDatabaseIDs(tx, ids) if err != nil { return fmt.Errorf("failed to get transaction IDs: %w", err) From 2f54475c026898dd3a52d829e8e88dd4181af396 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Thu, 18 Apr 2024 21:11:43 -0400 Subject: [PATCH 02/19] remove separate merkle proof table and just update merkle proofs alongside spent/resolved/etc status --- persist/sqlite/consensus.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 65e89965..9fe96025 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -466,7 +466,7 @@ func (ut *updateTx) AddSiacoinElements(bid types.BlockID, update consensusUpdate stmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements(output_id, block_id, leaf_index, merkle_proof, spent, source, maturity_height, address, value) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (output_id) - DO UPDATE SET spent = ?`) + DO UPDATE SET spent = ?, leaf_index = ?, merkle_proof = ?`) if err != nil { return nil, fmt.Errorf("addSiacoinElements: failed to prepare siacoin_elements statement: %w", err) } @@ -474,7 +474,7 @@ func (ut *updateTx) AddSiacoinElements(bid types.BlockID, update consensusUpdate scDBIds := make(map[types.SiacoinOutputID]int64) for _, sce := range newElements { - result, err := stmt.Exec(dbEncode(sce.StateElement.ID), dbEncode(bid), dbEncode(sce.StateElement.LeafIndex), dbEncode(sce.StateElement.MerkleProof), false, int(sources[types.SiacoinOutputID(sce.StateElement.ID)]), sce.MaturityHeight, dbEncode(sce.SiacoinOutput.Address), dbEncode(sce.SiacoinOutput.Value), false) + result, err := stmt.Exec(dbEncode(sce.StateElement.ID), dbEncode(bid), dbEncode(sce.StateElement.LeafIndex), dbEncode(sce.StateElement.MerkleProof), false, int(sources[types.SiacoinOutputID(sce.StateElement.ID)]), sce.MaturityHeight, dbEncode(sce.SiacoinOutput.Address), dbEncode(sce.SiacoinOutput.Value), false, dbEncode(sce.StateElement.LeafIndex), dbEncode(sce.StateElement.MerkleProof)) if err != nil { return nil, fmt.Errorf("addSiacoinElements: failed to execute siacoin_elements statement: %w", err) } @@ -487,7 +487,7 @@ func (ut *updateTx) AddSiacoinElements(bid types.BlockID, update consensusUpdate scDBIds[types.SiacoinOutputID(sce.StateElement.ID)] = dbID } for _, sce := range spentElements { - result, err := stmt.Exec(dbEncode(sce.StateElement.ID), dbEncode(bid), dbEncode(sce.StateElement.LeafIndex), dbEncode(sce.StateElement.MerkleProof), true, int(sources[types.SiacoinOutputID(sce.StateElement.ID)]), sce.MaturityHeight, dbEncode(sce.SiacoinOutput.Address), dbEncode(sce.SiacoinOutput.Value), true) + result, err := stmt.Exec(dbEncode(sce.StateElement.ID), dbEncode(bid), dbEncode(sce.StateElement.LeafIndex), dbEncode(sce.StateElement.MerkleProof), true, int(sources[types.SiacoinOutputID(sce.StateElement.ID)]), sce.MaturityHeight, dbEncode(sce.SiacoinOutput.Address), dbEncode(sce.SiacoinOutput.Value), true, dbEncode(sce.StateElement.LeafIndex), dbEncode(sce.StateElement.MerkleProof)) if err != nil { return nil, fmt.Errorf("addSiacoinElements: failed to execute siacoin_elements statement: %w", err) } @@ -507,7 +507,7 @@ func (ut *updateTx) AddSiafundElements(bid types.BlockID, spentElements, newElem stmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements(output_id, block_id, leaf_index, merkle_proof, spent, claim_start, address, value) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT - DO UPDATE SET spent = ?`) + DO UPDATE SET spent = ?, leaf_index = ?, merkle_proof = ?`) if err != nil { return nil, fmt.Errorf("addSiafundElements: failed to prepare siafund_elements statement: %w", err) } @@ -515,7 +515,7 @@ func (ut *updateTx) AddSiafundElements(bid types.BlockID, spentElements, newElem sfDBIds := make(map[types.SiafundOutputID]int64) for _, sfe := range newElements { - result, err := stmt.Exec(dbEncode(sfe.StateElement.ID), dbEncode(bid), dbEncode(sfe.StateElement.LeafIndex), dbEncode(sfe.StateElement.MerkleProof), false, dbEncode(sfe.ClaimStart), dbEncode(sfe.SiafundOutput.Address), dbEncode(sfe.SiafundOutput.Value), false) + result, err := stmt.Exec(dbEncode(sfe.StateElement.ID), dbEncode(bid), dbEncode(sfe.StateElement.LeafIndex), dbEncode(sfe.StateElement.MerkleProof), false, dbEncode(sfe.ClaimStart), dbEncode(sfe.SiafundOutput.Address), dbEncode(sfe.SiafundOutput.Value), false, dbEncode(sfe.StateElement.LeafIndex), dbEncode(sfe.StateElement.MerkleProof)) if err != nil { return nil, fmt.Errorf("addSiafundElements: failed to execute siafund_elements statement: %w", err) } @@ -528,7 +528,7 @@ func (ut *updateTx) AddSiafundElements(bid types.BlockID, spentElements, newElem sfDBIds[types.SiafundOutputID(sfe.StateElement.ID)] = dbID } for _, sfe := range spentElements { - result, err := stmt.Exec(dbEncode(sfe.StateElement.ID), dbEncode(bid), dbEncode(sfe.StateElement.LeafIndex), dbEncode(sfe.StateElement.MerkleProof), true, dbEncode(sfe.ClaimStart), dbEncode(sfe.SiafundOutput.Address), dbEncode(sfe.SiafundOutput.Value), true) + result, err := stmt.Exec(dbEncode(sfe.StateElement.ID), dbEncode(bid), dbEncode(sfe.StateElement.LeafIndex), dbEncode(sfe.StateElement.MerkleProof), true, dbEncode(sfe.ClaimStart), dbEncode(sfe.SiafundOutput.Address), dbEncode(sfe.SiafundOutput.Value), true, dbEncode(sfe.StateElement.LeafIndex), dbEncode(sfe.StateElement.MerkleProof)) if err != nil { return nil, fmt.Errorf("addSiafundElements: failed to execute siafund_elements statement: %w", err) } @@ -553,7 +553,7 @@ func (ut *updateTx) AddFileContractElements(bid types.BlockID, update consensusU stmt, err := ut.tx.Prepare(`INSERT INTO file_contract_elements(block_id, contract_id, leaf_index, merkle_proof, resolved, valid, filesize, file_merkle_root, window_start, window_end, payout, unlock_hash, revision_number) VALUES (?, ?, ?, ?, FALSE, TRUE, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (contract_id, revision_number) - DO UPDATE SET resolved = ?, valid = ? + DO UPDATE SET resolved = ?, valid = ?, leaf_index = ?, merkle_proof = ? RETURNING id;`) if err != nil { return nil, fmt.Errorf("addFileContractElements: failed to prepare file_contract_elements statement: %w", err) @@ -581,7 +581,7 @@ func (ut *updateTx) AddFileContractElements(bid types.BlockID, update consensusU } var dbID int64 - err := stmt.QueryRow(dbEncode(bid), dbEncode(fce.StateElement.ID), dbEncode(fce.StateElement.LeafIndex), dbEncode(fce.StateElement.MerkleProof), fc.Filesize, dbEncode(fc.FileMerkleRoot), fc.WindowStart, fc.WindowEnd, dbEncode(fc.Payout), dbEncode(fc.UnlockHash), fc.RevisionNumber, resolved, valid).Scan(&dbID) + err := stmt.QueryRow(dbEncode(bid), dbEncode(fce.StateElement.ID), dbEncode(fce.StateElement.LeafIndex), dbEncode(fce.StateElement.MerkleProof), fc.Filesize, dbEncode(fc.FileMerkleRoot), fc.WindowStart, fc.WindowEnd, dbEncode(fc.Payout), dbEncode(fc.UnlockHash), fc.RevisionNumber, resolved, valid, dbEncode(fce.StateElement.LeafIndex), dbEncode(fce.StateElement.MerkleProof)).Scan(&dbID) if err != nil { updateErr = fmt.Errorf("addFileContractElements: failed to execute file_contract_elements statement: %w", err) return From ceba22a899874c4aea147fa7e68304232962cdd3 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Fri, 19 Apr 2024 13:17:34 -0400 Subject: [PATCH 03/19] move higher level logic outside of persist/sqlite and into explorer package --- explorer/explorer.go | 4 +- persist/sqlite/consensus.go | 202 +++---------------------------- persist/sqlite/consensus_test.go | 2 +- 3 files changed, 20 insertions(+), 188 deletions(-) diff --git a/explorer/explorer.go b/explorer/explorer.go index a4a47bc6..1055f8c9 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -29,7 +29,7 @@ type ChainManager interface { // A Store is a database that stores information about elements, contracts, // and blocks. type Store interface { - ProcessChainUpdates(crus []chain.RevertUpdate, caus []chain.ApplyUpdate) error + UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error Tip() (types.ChainIndex, error) Block(id types.BlockID) (Block, error) @@ -56,7 +56,7 @@ func syncStore(store Store, cm ChainManager, index types.ChainIndex) error { return fmt.Errorf("failed to subscribe to chain manager: %w", err) } - if err := store.ProcessChainUpdates(crus, caus); err != nil { + if err := store.UpdateChainState(crus, caus); err != nil { return fmt.Errorf("failed to process updates: %w", err) } if len(crus) > 0 { diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 9fe96025..8a9642de 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -127,7 +127,7 @@ func (ut *updateTx) AddSiafundOutputs(id int64, txn types.Transaction, dbIDs map return nil } -func (ut *updateTx) AddFileContracts(id int64, txn types.Transaction, fcDBIds map[fileContract]int64) error { +func (ut *updateTx) AddFileContracts(id int64, txn types.Transaction, fcDBIds map[explorer.DBFileContract]int64) error { stmt, err := ut.tx.Prepare(`INSERT INTO transaction_file_contracts(transaction_id, transaction_order, contract_id) VALUES (?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContracts: failed to prepare statement: %w", err) @@ -147,7 +147,7 @@ func (ut *updateTx) AddFileContracts(id int64, txn types.Transaction, fcDBIds ma defer missedOutputsStmt.Close() for i := range txn.FileContracts { - dbID, ok := fcDBIds[fileContract{txn.FileContractID(i), 0}] + dbID, ok := fcDBIds[explorer.DBFileContract{txn.FileContractID(i), 0}] if !ok { return errors.New("addFileContracts: fcDbID not in map") } @@ -171,7 +171,7 @@ func (ut *updateTx) AddFileContracts(id int64, txn types.Transaction, fcDBIds ma return nil } -func (ut *updateTx) AddFileContractRevisions(id int64, txn types.Transaction, dbIDs map[fileContract]int64) error { +func (ut *updateTx) AddFileContractRevisions(id int64, txn types.Transaction, dbIDs map[explorer.DBFileContract]int64) error { stmt, err := ut.tx.Prepare(`INSERT INTO transaction_file_contract_revisions(transaction_id, transaction_order, contract_id, parent_id, unlock_conditions) VALUES (?, ?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContractRevisions: failed to prepare statement: %w", err) @@ -192,7 +192,7 @@ func (ut *updateTx) AddFileContractRevisions(id int64, txn types.Transaction, db for i := range txn.FileContractRevisions { fcr := &txn.FileContractRevisions[i] - dbID, ok := dbIDs[fileContract{fcr.ParentID, fcr.FileContract.RevisionNumber}] + dbID, ok := dbIDs[explorer.DBFileContract{fcr.ParentID, fcr.FileContract.RevisionNumber}] if !ok { return errors.New("addFileContractRevisions: dbID not in map") } @@ -217,7 +217,7 @@ func (ut *updateTx) AddFileContractRevisions(id int64, txn types.Transaction, db return nil } -func (ut *updateTx) AddTransactions(bid types.BlockID, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[fileContract]int64) error { +func (ut *updateTx) AddTransactions(bid types.BlockID, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[explorer.DBFileContract]int64) error { insertTransactionStmt, err := ut.tx.Prepare(`INSERT INTO transactions (transaction_id) VALUES (?) ON CONFLICT (transaction_id) DO UPDATE SET transaction_id=EXCLUDED.transaction_id -- technically a no-op, but necessary for the RETURNING clause RETURNING id;`) @@ -260,12 +260,6 @@ func (ut *updateTx) AddTransactions(bid types.BlockID, txns []types.Transaction, return nil } -type consensusUpdate interface { - ForEachSiacoinElement(fn func(sce types.SiacoinElement, spent bool)) - ForEachSiafundElement(fn func(sfe types.SiafundElement, spent bool)) - ForEachFileContractElement(fn func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool)) -} - type balance struct { sc types.Currency immatureSC types.Currency @@ -438,7 +432,7 @@ func (ut *updateTx) UpdateMaturedBalances(revert bool, height uint64) error { return nil } -func (ut *updateTx) AddSiacoinElements(bid types.BlockID, update consensusUpdate, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) { +func (ut *updateTx) AddSiacoinElements(bid types.BlockID, update explorer.ConsensusUpdate, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) { sources := make(map[types.SiacoinOutputID]explorer.Source) if applyUpdate, ok := update.(chain.ApplyUpdate); ok { block := applyUpdate.Block @@ -544,12 +538,7 @@ func (ut *updateTx) AddSiafundElements(bid types.BlockID, spentElements, newElem return sfDBIds, nil } -type fileContract struct { - id types.FileContractID - revisionNumber uint64 -} - -func (ut *updateTx) AddFileContractElements(bid types.BlockID, update consensusUpdate) (map[fileContract]int64, error) { +func (ut *updateTx) AddFileContractElements(bid types.BlockID, update explorer.ConsensusUpdate) (map[explorer.DBFileContract]int64, error) { stmt, err := ut.tx.Prepare(`INSERT INTO file_contract_elements(block_id, contract_id, leaf_index, merkle_proof, resolved, valid, filesize, file_merkle_root, window_start, window_end, payout, unlock_hash, revision_number) VALUES (?, ?, ?, ?, FALSE, TRUE, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (contract_id, revision_number) @@ -569,7 +558,7 @@ func (ut *updateTx) AddFileContractElements(bid types.BlockID, update consensusU } var updateErr error - fcDBIds := make(map[fileContract]int64) + fcDBIds := make(map[explorer.DBFileContract]int64) update.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { if updateErr != nil { return @@ -592,7 +581,7 @@ func (ut *updateTx) AddFileContractElements(bid types.BlockID, update consensusU return } - fcDBIds[fileContract{types.FileContractID(fce.StateElement.ID), fc.RevisionNumber}] = dbID + fcDBIds[explorer.DBFileContract{types.FileContractID(fce.StateElement.ID), fc.RevisionNumber}] = dbID }) return fcDBIds, updateErr } @@ -602,173 +591,16 @@ func (ut *updateTx) DeleteBlock(bid types.BlockID) error { return err } -// ProcessChainUpdates implements explorer.Store. -func (s *Store) ProcessChainUpdates(crus []chain.RevertUpdate, caus []chain.ApplyUpdate) error { - return s.transaction(func(dbTxn *txn) error { - tx := &updateTx{tx: dbTxn} - - for _, cru := range crus { - if err := tx.UpdateMaturedBalances(true, cru.State.Index.Height+1); err != nil { - return fmt.Errorf("revertUpdate: failed to update matured balances: %w", err) - } - - created := make(map[types.Hash256]bool) - ephemeral := make(map[types.Hash256]bool) - for _, txn := range cru.Block.Transactions { - for i := range txn.SiacoinOutputs { - created[types.Hash256(txn.SiacoinOutputID(i))] = true - } - for _, input := range txn.SiacoinInputs { - ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] - } - for i := range txn.SiafundOutputs { - created[types.Hash256(txn.SiafundOutputID(i))] = true - } - for _, input := range txn.SiafundInputs { - ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] - } - } - - // add new siacoin elements to the store - var newSiacoinElements, spentSiacoinElements []types.SiacoinElement - var ephemeralSiacoinElements []types.SiacoinElement - cru.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { - if ephemeral[se.ID] { - ephemeralSiacoinElements = append(ephemeralSiacoinElements, se) - return - } - - if spent { - newSiacoinElements = append(newSiacoinElements, se) - } else { - spentSiacoinElements = append(spentSiacoinElements, se) - } - }) - - var newSiafundElements, spentSiafundElements []types.SiafundElement - var ephemeralSiafundElements []types.SiafundElement - cru.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { - if ephemeral[se.ID] { - ephemeralSiafundElements = append(ephemeralSiafundElements, se) - return - } - - if spent { - newSiafundElements = append(newSiafundElements, se) - } else { - spentSiafundElements = append(spentSiafundElements, se) - } - }) - - // log.Println("REVERT!") - if _, err := tx.AddSiacoinElements( - cru.Block.ID(), - cru, - spentSiacoinElements, - append(newSiacoinElements, ephemeralSiacoinElements...), - ); err != nil { - return fmt.Errorf("revertUpdate: failed to update siacoin output state: %w", err) - } else if _, err := tx.AddSiafundElements( - cru.Block.ID(), - spentSiafundElements, - append(newSiafundElements, ephemeralSiafundElements...), - ); err != nil { - return fmt.Errorf("revertUpdate: failed to update siafund output state: %w", err) - } else if err := tx.UpdateBalances(cru.State.Index.Height+1, spentSiacoinElements, newSiacoinElements, spentSiafundElements, newSiafundElements); err != nil { - return fmt.Errorf("revertUpdate: failed to update balances: %w", err) - } else if _, err := tx.AddFileContractElements(cru.Block.ID(), cru); err != nil { - return fmt.Errorf("revertUpdate: failed to update file contract state: %w", err) - } else if err := tx.DeleteBlock(cru.Block.ID()); err != nil { - return fmt.Errorf("revertUpdate: failed to delete block: %w", err) - } +// UpdateChainState implements explorer.Store +func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error { + return s.transaction(func(tx *txn) error { + utx := &updateTx{ + tx: tx, + relevantAddresses: make(map[types.Address]bool), } - for _, cau := range caus { - if err := tx.AddBlock(cau.Block, cau.State.Index.Height); err != nil { - return fmt.Errorf("applyUpdates: failed to add block: %w", err) - } else if err := tx.UpdateMaturedBalances(false, cau.State.Index.Height); err != nil { - return fmt.Errorf("applyUpdates: failed to update matured balances: %w", err) - } - - created := make(map[types.Hash256]bool) - ephemeral := make(map[types.Hash256]bool) - for _, txn := range cau.Block.Transactions { - for i := range txn.SiacoinOutputs { - created[types.Hash256(txn.SiacoinOutputID(i))] = true - } - for _, input := range txn.SiacoinInputs { - ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] - } - for i := range txn.SiafundOutputs { - created[types.Hash256(txn.SiafundOutputID(i))] = true - } - for _, input := range txn.SiafundInputs { - ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] - } - } - - // add new siacoin elements to the store - var newSiacoinElements, spentSiacoinElements []types.SiacoinElement - var ephemeralSiacoinElements []types.SiacoinElement - cau.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { - if ephemeral[se.ID] { - ephemeralSiacoinElements = append(ephemeralSiacoinElements, se) - return - } - - if spent { - spentSiacoinElements = append(spentSiacoinElements, se) - } else { - newSiacoinElements = append(newSiacoinElements, se) - } - }) - - var newSiafundElements, spentSiafundElements []types.SiafundElement - var ephemeralSiafundElements []types.SiafundElement - cau.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { - if ephemeral[se.ID] { - ephemeralSiafundElements = append(ephemeralSiafundElements, se) - return - } - - if spent { - spentSiafundElements = append(spentSiafundElements, se) - } else { - newSiafundElements = append(newSiafundElements, se) - } - }) - - scDBIds, err := tx.AddSiacoinElements( - cau.Block.ID(), - cau, - append(spentSiacoinElements, ephemeralSiacoinElements...), - newSiacoinElements, - ) - if err != nil { - return fmt.Errorf("applyUpdates: failed to add siacoin outputs: %w", err) - } - sfDBIds, err := tx.AddSiafundElements( - cau.Block.ID(), - append(spentSiafundElements, ephemeralSiafundElements...), - newSiafundElements, - ) - if err != nil { - return fmt.Errorf("applyUpdates: failed to add siafund outputs: %w", err) - } - if err := tx.UpdateBalances(cau.State.Index.Height, spentSiacoinElements, newSiacoinElements, spentSiafundElements, newSiafundElements); err != nil { - return fmt.Errorf("applyUpdates: failed to update balances: %w", err) - } - - fcDBIds, err := tx.AddFileContractElements(cau.Block.ID(), cau) - if err != nil { - return fmt.Errorf("applyUpdates: failed to add file contracts: %w", err) - } - - if err := tx.AddMinerPayouts(cau.Block.ID(), cau.State.Index.Height, cau.Block.MinerPayouts, scDBIds); err != nil { - return fmt.Errorf("applyUpdates: failed to add miner payouts: %w", err) - } else if err := tx.AddTransactions(cau.Block.ID(), cau.Block.Transactions, scDBIds, sfDBIds, fcDBIds); err != nil { - return fmt.Errorf("applyUpdates: failed to add transactions: addTransactions: %w", err) - } + if err := explorer.UpdateChainState(utx, reverted, applied); err != nil { + return fmt.Errorf("failed to update chain state: %w", err) } return nil }) diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index 8c3391e7..ad66a09b 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -126,7 +126,7 @@ func syncDB(t *testing.T, db *sqlite.Store, cm *chain.Manager) { t.Fatal(err) } - if err := db.ProcessChainUpdates(crus, caus); err != nil { + if err := db.UpdateChainState(crus, caus); err != nil { t.Fatal("failed to process updates:", err) } if len(crus) > 0 { From 9c73a6d69aa580527dff7a31d24909e8d13ba341 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Fri, 19 Apr 2024 13:32:37 -0400 Subject: [PATCH 04/19] use separate slice encoding functions, rename dbEncode -> encode, dbDecode -> decode --- persist/sqlite/blocks.go | 4 +- persist/sqlite/consensus.go | 54 ++++++++++----------- persist/sqlite/contracts.go | 4 +- persist/sqlite/encoding.go | 86 +++++++++++++++++++++++----------- persist/sqlite/outputs.go | 10 ++-- persist/sqlite/transactions.go | 24 +++++----- 6 files changed, 107 insertions(+), 75 deletions(-) diff --git a/persist/sqlite/blocks.go b/persist/sqlite/blocks.go index 45c1338a..fad93454 100644 --- a/persist/sqlite/blocks.go +++ b/persist/sqlite/blocks.go @@ -10,7 +10,7 @@ import ( // Block implements explorer.Store. func (s *Store) Block(id types.BlockID) (result explorer.Block, err error) { err = s.transaction(func(tx *txn) error { - err = tx.QueryRow(`SELECT parent_id, nonce, timestamp, height FROM blocks WHERE id=?`, dbEncode(id)).Scan(dbDecode(&result.ParentID), dbDecode(&result.Nonce), dbDecode(&result.Timestamp), &result.Height) + err = tx.QueryRow(`SELECT parent_id, nonce, timestamp, height FROM blocks WHERE id=?`, encode(id)).Scan(decode(&result.ParentID), decode(&result.Nonce), decode(&result.Timestamp), &result.Height) if err != nil { return err } @@ -39,7 +39,7 @@ func (s *Store) Block(id types.BlockID) (result explorer.Block, err error) { // BestTip implements explorer.Store. func (s *Store) BestTip(height uint64) (result types.ChainIndex, err error) { err = s.transaction(func(tx *txn) error { - err = tx.QueryRow(`SELECT id, height FROM blocks WHERE height=?`, height).Scan(dbDecode(&result.ID), dbDecode(&result.Height)) + err = tx.QueryRow(`SELECT id, height FROM blocks WHERE height=?`, height).Scan(decode(&result.ID), decode(&result.Height)) if err != nil { return err } diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 8a9642de..d982c28c 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -17,7 +17,7 @@ type updateTx struct { func (ut *updateTx) AddBlock(b types.Block, height uint64) error { // nonce is encoded because database/sql doesn't support uint64 with high bit set - _, err := ut.tx.Exec("INSERT INTO blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", dbEncode(b.ID()), height, dbEncode(b.ParentID), dbEncode(b.Nonce), dbEncode(b.Timestamp)) + _, err := ut.tx.Exec("INSERT INTO blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encode(b.Nonce), encode(b.Timestamp)) return err } @@ -34,7 +34,7 @@ func (ut *updateTx) AddMinerPayouts(bid types.BlockID, height uint64, scos []typ return errors.New("addMinerPayouts: dbID not in map") } - if _, err := stmt.Exec(dbEncode(bid), i, dbID); err != nil { + if _, err := stmt.Exec(encode(bid), i, dbID); err != nil { return fmt.Errorf("addMinerPayouts: failed to execute statement: %w", err) } } @@ -65,7 +65,7 @@ func (ut *updateTx) AddSiacoinInputs(id int64, txn types.Transaction) error { defer stmt.Close() for i, sci := range txn.SiacoinInputs { - if _, err := stmt.Exec(id, i, dbEncode(sci.ParentID), dbEncode(sci.UnlockConditions)); err != nil { + if _, err := stmt.Exec(id, i, encode(sci.ParentID), encode(sci.UnlockConditions)); err != nil { return fmt.Errorf("addSiacoinInputs: failed to execute statement: %w", err) } } @@ -100,7 +100,7 @@ func (ut *updateTx) AddSiafundInputs(id int64, txn types.Transaction) error { defer stmt.Close() for i, sci := range txn.SiafundInputs { - if _, err := stmt.Exec(id, i, dbEncode(sci.ParentID), dbEncode(sci.UnlockConditions), dbEncode(sci.ClaimAddress)); err != nil { + if _, err := stmt.Exec(id, i, encode(sci.ParentID), encode(sci.UnlockConditions), encode(sci.ClaimAddress)); err != nil { return fmt.Errorf("addSiafundInputs: failed to execute statement: %w", err) } } @@ -157,13 +157,13 @@ func (ut *updateTx) AddFileContracts(id int64, txn types.Transaction, fcDBIds ma } for j, sco := range txn.FileContracts[i].ValidProofOutputs { - if _, err := validOutputsStmt.Exec(dbID, j, dbEncode(sco.Address), dbEncode(sco.Value)); err != nil { + if _, err := validOutputsStmt.Exec(dbID, j, encode(sco.Address), encode(sco.Value)); err != nil { return fmt.Errorf("addFileContracts: failed to execute valid proof outputs statement: %w", err) } } for j, sco := range txn.FileContracts[i].MissedProofOutputs { - if _, err := missedOutputsStmt.Exec(dbID, j, dbEncode(sco.Address), dbEncode(sco.Value)); err != nil { + if _, err := missedOutputsStmt.Exec(dbID, j, encode(sco.Address), encode(sco.Value)); err != nil { return fmt.Errorf("addFileContracts: failed to execute missed proof outputs statement: %w", err) } } @@ -197,18 +197,18 @@ func (ut *updateTx) AddFileContractRevisions(id int64, txn types.Transaction, db return errors.New("addFileContractRevisions: dbID not in map") } - if _, err := stmt.Exec(id, i, dbID, dbEncode(fcr.ParentID), dbEncode(fcr.UnlockConditions)); err != nil { + if _, err := stmt.Exec(id, i, dbID, encode(fcr.ParentID), encode(fcr.UnlockConditions)); err != nil { return fmt.Errorf("addFileContractRevisions: failed to execute statement: %w", err) } for j, sco := range txn.FileContractRevisions[i].ValidProofOutputs { - if _, err := validOutputsStmt.Exec(dbID, j, dbEncode(sco.Address), dbEncode(sco.Value)); err != nil { + if _, err := validOutputsStmt.Exec(dbID, j, encode(sco.Address), encode(sco.Value)); err != nil { return fmt.Errorf("addFileContractRevisions: failed to execute valid proof outputs statement: %w", err) } } for j, sco := range txn.FileContractRevisions[i].MissedProofOutputs { - if _, err := missedOutputsStmt.Exec(dbID, j, dbEncode(sco.Address), dbEncode(sco.Value)); err != nil { + if _, err := missedOutputsStmt.Exec(dbID, j, encode(sco.Address), encode(sco.Value)); err != nil { return fmt.Errorf("addFileContractRevisions: failed to execute missed proof outputs statement: %w", err) } } @@ -234,12 +234,12 @@ func (ut *updateTx) AddTransactions(bid types.BlockID, txns []types.Transaction, for i, txn := range txns { var txnID int64 - err := insertTransactionStmt.QueryRow(dbEncode(txn.ID())).Scan(&txnID) + err := insertTransactionStmt.QueryRow(encode(txn.ID())).Scan(&txnID) if err != nil { return fmt.Errorf("failed to insert into transactions: %w", err) } - if _, err := blockTransactionsStmt.Exec(dbEncode(bid), txnID, i); err != nil { + if _, err := blockTransactionsStmt.Exec(encode(bid), txnID, i); err != nil { return fmt.Errorf("failed to insert into block_transactions: %w", err) } else if err := ut.AddArbitraryData(txnID, txn); err != nil { return fmt.Errorf("failed to add arbitrary data: %w", err) @@ -283,7 +283,7 @@ func (ut *updateTx) UpdateBalances(height uint64, spentSiacoinElements, newSiaco var addressList []any for address := range addresses { - addressList = append(addressList, dbEncode(address)) + addressList = append(addressList, encode(address)) } rows, err := ut.tx.Query(`SELECT address, siacoin_balance, immature_siacoin_balance, siafund_balance @@ -297,7 +297,7 @@ func (ut *updateTx) UpdateBalances(height uint64, spentSiacoinElements, newSiaco for rows.Next() { var bal balance var address types.Address - if err := rows.Scan(dbDecode(&address), dbDecode(&bal.sc), dbDecode(&bal.immatureSC), dbDecode(&bal.sf)); err != nil { + if err := rows.Scan(decode(&address), decode(&bal.sc), decode(&bal.immatureSC), decode(&bal.sf)); err != nil { return err } addresses[address] = bal @@ -346,7 +346,7 @@ func (ut *updateTx) UpdateBalances(height uint64, spentSiacoinElements, newSiaco defer stmt.Close() for addr, bal := range addresses { - if _, err := stmt.Exec(dbEncode(addr), dbEncode(bal.sc), dbEncode(bal.immatureSC), dbEncode(bal.sf), dbEncode(bal.sc), dbEncode(bal.immatureSC), dbEncode(bal.sf)); err != nil { + if _, err := stmt.Exec(encode(addr), encode(bal.sc), encode(bal.immatureSC), encode(bal.sf), encode(bal.sc), encode(bal.immatureSC), encode(bal.sf)); err != nil { return fmt.Errorf("updateBalances: failed to exec statement: %w", err) } // log.Println(addr, "=", bal.sc) @@ -374,11 +374,11 @@ func (ut *updateTx) UpdateMaturedBalances(revert bool, height uint64) error { var scos []types.SiacoinOutput for rows.Next() { var sco types.SiacoinOutput - if err := rows.Scan(dbDecode(&sco.Address), dbDecode(&sco.Value)); err != nil { + if err := rows.Scan(decode(&sco.Address), decode(&sco.Value)); err != nil { return fmt.Errorf("updateMaturedBalances: failed to scan maturing outputs: %w", err) } scos = append(scos, sco) - addressList = append(addressList, dbEncode(sco.Address)) + addressList = append(addressList, encode(sco.Address)) } balanceRows, err := ut.tx.Query(`SELECT address, siacoin_balance, immature_siacoin_balance @@ -393,7 +393,7 @@ func (ut *updateTx) UpdateMaturedBalances(revert bool, height uint64) error { for balanceRows.Next() { var address types.Address var bal balance - if err := balanceRows.Scan(dbDecode(&address), dbDecode(&bal.sc), dbDecode(&bal.immatureSC)); err != nil { + if err := balanceRows.Scan(decode(&address), decode(&bal.sc), decode(&bal.immatureSC)); err != nil { return fmt.Errorf("updateMaturedBalances: failed to scan balance: %w", err) } addresses[address] = bal @@ -422,9 +422,9 @@ func (ut *updateTx) UpdateMaturedBalances(revert bool, height uint64) error { } defer stmt.Close() - initialSF := dbEncode(uint64(0)) + initialSF := encode(uint64(0)) for addr, bal := range addresses { - if _, err := stmt.Exec(dbEncode(addr), dbEncode(bal.sc), dbEncode(bal.immatureSC), initialSF, dbEncode(bal.sc), dbEncode(bal.immatureSC)); err != nil { + if _, err := stmt.Exec(encode(addr), encode(bal.sc), encode(bal.immatureSC), initialSF, encode(bal.sc), encode(bal.immatureSC)); err != nil { return fmt.Errorf("updateMaturedBalances: failed to exec statement: %w", err) } } @@ -468,7 +468,7 @@ func (ut *updateTx) AddSiacoinElements(bid types.BlockID, update explorer.Consen scDBIds := make(map[types.SiacoinOutputID]int64) for _, sce := range newElements { - result, err := stmt.Exec(dbEncode(sce.StateElement.ID), dbEncode(bid), dbEncode(sce.StateElement.LeafIndex), dbEncode(sce.StateElement.MerkleProof), false, int(sources[types.SiacoinOutputID(sce.StateElement.ID)]), sce.MaturityHeight, dbEncode(sce.SiacoinOutput.Address), dbEncode(sce.SiacoinOutput.Value), false, dbEncode(sce.StateElement.LeafIndex), dbEncode(sce.StateElement.MerkleProof)) + result, err := stmt.Exec(encode(sce.StateElement.ID), encode(bid), encode(sce.StateElement.LeafIndex), encodeSlice(sce.StateElement.MerkleProof), false, int(sources[types.SiacoinOutputID(sce.StateElement.ID)]), sce.MaturityHeight, encode(sce.SiacoinOutput.Address), encode(sce.SiacoinOutput.Value), false, encode(sce.StateElement.LeafIndex), encodeSlice(sce.StateElement.MerkleProof)) if err != nil { return nil, fmt.Errorf("addSiacoinElements: failed to execute siacoin_elements statement: %w", err) } @@ -481,7 +481,7 @@ func (ut *updateTx) AddSiacoinElements(bid types.BlockID, update explorer.Consen scDBIds[types.SiacoinOutputID(sce.StateElement.ID)] = dbID } for _, sce := range spentElements { - result, err := stmt.Exec(dbEncode(sce.StateElement.ID), dbEncode(bid), dbEncode(sce.StateElement.LeafIndex), dbEncode(sce.StateElement.MerkleProof), true, int(sources[types.SiacoinOutputID(sce.StateElement.ID)]), sce.MaturityHeight, dbEncode(sce.SiacoinOutput.Address), dbEncode(sce.SiacoinOutput.Value), true, dbEncode(sce.StateElement.LeafIndex), dbEncode(sce.StateElement.MerkleProof)) + result, err := stmt.Exec(encode(sce.StateElement.ID), encode(bid), encode(sce.StateElement.LeafIndex), encodeSlice(sce.StateElement.MerkleProof), true, int(sources[types.SiacoinOutputID(sce.StateElement.ID)]), sce.MaturityHeight, encode(sce.SiacoinOutput.Address), encode(sce.SiacoinOutput.Value), true, encode(sce.StateElement.LeafIndex), encodeSlice(sce.StateElement.MerkleProof)) if err != nil { return nil, fmt.Errorf("addSiacoinElements: failed to execute siacoin_elements statement: %w", err) } @@ -509,7 +509,7 @@ func (ut *updateTx) AddSiafundElements(bid types.BlockID, spentElements, newElem sfDBIds := make(map[types.SiafundOutputID]int64) for _, sfe := range newElements { - result, err := stmt.Exec(dbEncode(sfe.StateElement.ID), dbEncode(bid), dbEncode(sfe.StateElement.LeafIndex), dbEncode(sfe.StateElement.MerkleProof), false, dbEncode(sfe.ClaimStart), dbEncode(sfe.SiafundOutput.Address), dbEncode(sfe.SiafundOutput.Value), false, dbEncode(sfe.StateElement.LeafIndex), dbEncode(sfe.StateElement.MerkleProof)) + result, err := stmt.Exec(encode(sfe.StateElement.ID), encode(bid), encode(sfe.StateElement.LeafIndex), encodeSlice(sfe.StateElement.MerkleProof), false, encode(sfe.ClaimStart), encode(sfe.SiafundOutput.Address), encode(sfe.SiafundOutput.Value), false, encode(sfe.StateElement.LeafIndex), encodeSlice(sfe.StateElement.MerkleProof)) if err != nil { return nil, fmt.Errorf("addSiafundElements: failed to execute siafund_elements statement: %w", err) } @@ -522,7 +522,7 @@ func (ut *updateTx) AddSiafundElements(bid types.BlockID, spentElements, newElem sfDBIds[types.SiafundOutputID(sfe.StateElement.ID)] = dbID } for _, sfe := range spentElements { - result, err := stmt.Exec(dbEncode(sfe.StateElement.ID), dbEncode(bid), dbEncode(sfe.StateElement.LeafIndex), dbEncode(sfe.StateElement.MerkleProof), true, dbEncode(sfe.ClaimStart), dbEncode(sfe.SiafundOutput.Address), dbEncode(sfe.SiafundOutput.Value), true, dbEncode(sfe.StateElement.LeafIndex), dbEncode(sfe.StateElement.MerkleProof)) + result, err := stmt.Exec(encode(sfe.StateElement.ID), encode(bid), encode(sfe.StateElement.LeafIndex), encodeSlice(sfe.StateElement.MerkleProof), true, encode(sfe.ClaimStart), encode(sfe.SiafundOutput.Address), encode(sfe.SiafundOutput.Value), true, encode(sfe.StateElement.LeafIndex), encodeSlice(sfe.StateElement.MerkleProof)) if err != nil { return nil, fmt.Errorf("addSiafundElements: failed to execute siafund_elements statement: %w", err) } @@ -570,13 +570,13 @@ func (ut *updateTx) AddFileContractElements(bid types.BlockID, update explorer.C } var dbID int64 - err := stmt.QueryRow(dbEncode(bid), dbEncode(fce.StateElement.ID), dbEncode(fce.StateElement.LeafIndex), dbEncode(fce.StateElement.MerkleProof), fc.Filesize, dbEncode(fc.FileMerkleRoot), fc.WindowStart, fc.WindowEnd, dbEncode(fc.Payout), dbEncode(fc.UnlockHash), fc.RevisionNumber, resolved, valid, dbEncode(fce.StateElement.LeafIndex), dbEncode(fce.StateElement.MerkleProof)).Scan(&dbID) + err := stmt.QueryRow(encode(bid), encode(fce.StateElement.ID), encode(fce.StateElement.LeafIndex), encodeSlice(fce.StateElement.MerkleProof), fc.Filesize, encode(fc.FileMerkleRoot), fc.WindowStart, fc.WindowEnd, encode(fc.Payout), encode(fc.UnlockHash), fc.RevisionNumber, resolved, valid, encode(fce.StateElement.LeafIndex), encodeSlice(fce.StateElement.MerkleProof)).Scan(&dbID) if err != nil { updateErr = fmt.Errorf("addFileContractElements: failed to execute file_contract_elements statement: %w", err) return } - if _, err := revisionStmt.Exec(dbEncode(fce.StateElement.ID), dbID, dbID); err != nil { + if _, err := revisionStmt.Exec(encode(fce.StateElement.ID), dbID, dbID); err != nil { updateErr = fmt.Errorf("addFileContractElements: failed to update last revision number: %w", err) return } @@ -587,7 +587,7 @@ func (ut *updateTx) AddFileContractElements(bid types.BlockID, update explorer.C } func (ut *updateTx) DeleteBlock(bid types.BlockID) error { - _, err := ut.tx.Exec("DELETE FROM blocks WHERE id = ?", dbEncode(bid)) + _, err := ut.tx.Exec("DELETE FROM blocks WHERE id = ?", encode(bid)) return err } @@ -610,7 +610,7 @@ func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain. func (s *Store) Tip() (result types.ChainIndex, err error) { const query = `SELECT id, height FROM blocks ORDER BY height DESC LIMIT 1` err = s.transaction(func(dbTxn *txn) error { - return dbTxn.QueryRow(query).Scan(dbDecode(&result.ID), &result.Height) + return dbTxn.QueryRow(query).Scan(decode(&result.ID), &result.Height) }) if errors.Is(err, sql.ErrNoRows) { return types.ChainIndex{}, explorer.ErrNoTip diff --git a/persist/sqlite/contracts.go b/persist/sqlite/contracts.go index 4a5394ba..386bdd22 100644 --- a/persist/sqlite/contracts.go +++ b/persist/sqlite/contracts.go @@ -12,7 +12,7 @@ func (s *Store) Contracts(ids []types.FileContractID) (result []explorer.FileCon encodedIDs := func(ids []types.FileContractID) []any { result := make([]any, len(ids)) for i, id := range ids { - result[i] = dbEncode(id) + result[i] = encode(id) } return result } @@ -33,7 +33,7 @@ func (s *Store) Contracts(ids []types.FileContractID) (result []explorer.FileCon for rows.Next() { var contractID int64 var fc explorer.FileContract - if err := rows.Scan(&contractID, dbDecode(&fc.StateElement.ID), dbDecode(&fc.StateElement.LeafIndex), dbDecode(&fc.StateElement.MerkleProof), &fc.Resolved, &fc.Valid, &fc.Filesize, dbDecode(&fc.FileMerkleRoot), &fc.WindowStart, &fc.WindowEnd, dbDecode(&fc.Payout), dbDecode(&fc.UnlockHash), &fc.RevisionNumber); err != nil { + if err := rows.Scan(&contractID, decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), decodeSlice(&fc.StateElement.MerkleProof), &fc.Resolved, &fc.Valid, &fc.Filesize, decode(&fc.FileMerkleRoot), &fc.WindowStart, &fc.WindowEnd, decode(&fc.Payout), decode(&fc.UnlockHash), &fc.RevisionNumber); err != nil { return fmt.Errorf("failed to scan transaction: %w", err) } diff --git a/persist/sqlite/encoding.go b/persist/sqlite/encoding.go index 90b42f7b..966c6b00 100644 --- a/persist/sqlite/encoding.go +++ b/persist/sqlite/encoding.go @@ -11,31 +11,23 @@ import ( "go.sia.tech/core/types" ) -func dbEncode(obj any) any { +func encode(obj any) any { switch obj := obj.(type) { + case types.Currency: + // Currency is encoded as two 64-bit big-endian integers for sorting + buf := make([]byte, 16) + binary.BigEndian.PutUint64(buf, obj.Hi) + binary.BigEndian.PutUint64(buf[8:], obj.Lo) + return buf case types.EncoderTo: var buf bytes.Buffer e := types.NewEncoder(&buf) obj.EncodeTo(e) e.Flush() return buf.Bytes() - case []types.Hash256: - var buf bytes.Buffer - e := types.NewEncoder(&buf) - e.WritePrefix(len(obj)) - for _, o := range obj { - o.EncodeTo(e) - } - e.Flush() - return buf.Bytes() - case types.Currency: - buf := make([]byte, 16) - binary.BigEndian.PutUint64(buf[:8], obj.Hi) - binary.BigEndian.PutUint64(buf[8:], obj.Lo) - return buf case uint64: b := make([]byte, 8) - binary.BigEndian.PutUint64(b, obj) + binary.LittleEndian.PutUint64(b, obj) return b case time.Time: return obj.Unix() @@ -57,21 +49,18 @@ func (d *decodable) Scan(src any) error { switch src := src.(type) { case []byte: switch v := d.v.(type) { + case *types.Currency: + if len(src) != 16 { + return fmt.Errorf("cannot scan %d bytes into Currency", len(src)) + } + v.Hi = binary.BigEndian.Uint64(src) + v.Lo = binary.BigEndian.Uint64(src[8:]) case types.DecoderFrom: dec := types.NewBufDecoder(src) v.DecodeFrom(dec) return dec.Err() - case *[]types.Hash256: - dec := types.NewBufDecoder(src) - *v = make([]types.Hash256, dec.ReadPrefix()) - for i := range *v { - (*v)[i].DecodeFrom(dec) - } - case *types.Currency: - v.Hi = binary.BigEndian.Uint64(src[:8]) - v.Lo = binary.BigEndian.Uint64(src[8:]) case *uint64: - *v = binary.BigEndian.Uint64(src) + *v = binary.LittleEndian.Uint64(src) default: return fmt.Errorf("cannot scan %T to %T", src, d.v) } @@ -91,6 +80,49 @@ func (d *decodable) Scan(src any) error { } } -func dbDecode(obj any) sql.Scanner { +func decode(obj any) sql.Scanner { return &decodable{obj} } + +type decodableSlice[T any] struct { + v *[]T +} + +func (d *decodableSlice[T]) Scan(src any) error { + switch src := src.(type) { + case []byte: + dec := types.NewBufDecoder(src) + s := make([]T, dec.ReadPrefix()) + for i := range s { + dv, ok := any(&s[i]).(types.DecoderFrom) + if !ok { + panic(fmt.Errorf("cannot decode %T", s[i])) + } + dv.DecodeFrom(dec) + } + if err := dec.Err(); err != nil { + return err + } + *d.v = s + return nil + default: + return fmt.Errorf("cannot scan %T to []byte", src) + } +} + +func decodeSlice[T any](v *[]T) sql.Scanner { + return &decodableSlice[T]{v: v} +} + +func encodeSlice[T types.EncoderTo](v []T) []byte { + var buf bytes.Buffer + enc := types.NewEncoder(&buf) + enc.WritePrefix(len(v)) + for _, e := range v { + e.EncodeTo(enc) + } + if err := enc.Flush(); err != nil { + panic(err) + } + return buf.Bytes() +} diff --git a/persist/sqlite/outputs.go b/persist/sqlite/outputs.go index eb399cb6..a5c9e392 100644 --- a/persist/sqlite/outputs.go +++ b/persist/sqlite/outputs.go @@ -11,7 +11,7 @@ import ( // UnspentSiacoinOutputs implements explorer.Store. func (s *Store) UnspentSiacoinOutputs(address types.Address, limit, offset uint64) (result []explorer.SiacoinOutput, err error) { err = s.transaction(func(tx *txn) error { - rows, err := tx.Query(`SELECT output_id, leaf_index, merkle_proof, source, maturity_height, address, value FROM siacoin_elements WHERE address = ? AND spent = 0 LIMIT ? OFFSET ?`, dbEncode(address), limit, offset) + rows, err := tx.Query(`SELECT output_id, leaf_index, merkle_proof, source, maturity_height, address, value FROM siacoin_elements WHERE address = ? AND spent = 0 LIMIT ? OFFSET ?`, encode(address), limit, offset) if err != nil { return fmt.Errorf("failed to query siacoin outputs: %w", err) } @@ -19,7 +19,7 @@ func (s *Store) UnspentSiacoinOutputs(address types.Address, limit, offset uint6 for rows.Next() { var sco explorer.SiacoinOutput - if err := rows.Scan(dbDecode(&sco.StateElement.ID), dbDecode(&sco.StateElement.LeafIndex), dbDecode(&sco.StateElement.MerkleProof), &sco.Source, &sco.MaturityHeight, dbDecode(&sco.SiacoinOutput.Address), dbDecode(&sco.SiacoinOutput.Value)); err != nil { + if err := rows.Scan(decode(&sco.StateElement.ID), decode(&sco.StateElement.LeafIndex), decodeSlice(&sco.StateElement.MerkleProof), &sco.Source, &sco.MaturityHeight, decode(&sco.SiacoinOutput.Address), decode(&sco.SiacoinOutput.Value)); err != nil { return fmt.Errorf("failed to scan siacoin output: %w", err) } result = append(result, sco) @@ -32,7 +32,7 @@ func (s *Store) UnspentSiacoinOutputs(address types.Address, limit, offset uint6 // UnspentSiafundOutputs implements explorer.Store. func (s *Store) UnspentSiafundOutputs(address types.Address, limit, offset uint64) (result []explorer.SiafundOutput, err error) { err = s.transaction(func(tx *txn) error { - rows, err := tx.Query(`SELECT output_id, leaf_index, merkle_proof, claim_start, address, value FROM siafund_elements WHERE address = ? AND spent = 0 LIMIT ? OFFSET ?`, dbEncode(address), limit, offset) + rows, err := tx.Query(`SELECT output_id, leaf_index, merkle_proof, claim_start, address, value FROM siafund_elements WHERE address = ? AND spent = 0 LIMIT ? OFFSET ?`, encode(address), limit, offset) if err != nil { return fmt.Errorf("failed to query siafund outputs: %w", err) } @@ -40,7 +40,7 @@ func (s *Store) UnspentSiafundOutputs(address types.Address, limit, offset uint6 for rows.Next() { var sfo explorer.SiafundOutput - if err := rows.Scan(dbDecode(&sfo.StateElement.ID), dbDecode(&sfo.StateElement.LeafIndex), dbDecode(&sfo.StateElement.MerkleProof), dbDecode(&sfo.ClaimStart), dbDecode(&sfo.SiafundOutput.Address), dbDecode(&sfo.SiafundOutput.Value)); err != nil { + if err := rows.Scan(decode(&sfo.StateElement.ID), decode(&sfo.StateElement.LeafIndex), decodeSlice(&sfo.StateElement.MerkleProof), decode(&sfo.ClaimStart), decode(&sfo.SiafundOutput.Address), decode(&sfo.SiafundOutput.Value)); err != nil { return fmt.Errorf("failed to scan siafund output: %w", err) } result = append(result, sfo) @@ -53,7 +53,7 @@ func (s *Store) UnspentSiafundOutputs(address types.Address, limit, offset uint6 // Balance implements explorer.Store. func (s *Store) Balance(address types.Address) (sc types.Currency, immatureSC types.Currency, sf uint64, err error) { err = s.transaction(func(tx *txn) error { - err = tx.QueryRow(`SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM address_balance WHERE address = ?`, dbEncode(address)).Scan(dbDecode(&sc), dbDecode(&immatureSC), dbDecode(&sf)) + err = tx.QueryRow(`SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM address_balance WHERE address = ?`, encode(address)).Scan(decode(&sc), decode(&immatureSC), decode(&sf)) if err == sql.ErrNoRows { return nil } else if err != nil { diff --git a/persist/sqlite/transactions.go b/persist/sqlite/transactions.go index 57e33614..a9f7cbca 100644 --- a/persist/sqlite/transactions.go +++ b/persist/sqlite/transactions.go @@ -49,7 +49,7 @@ ORDER BY ts.transaction_order ASC` for rows.Next() { var txnID int64 var sco explorer.SiacoinOutput - if err := rows.Scan(&txnID, dbDecode(&sco.StateElement.ID), dbDecode(&sco.LeafIndex), dbDecode(&sco.MerkleProof), &sco.Source, &sco.MaturityHeight, dbDecode(&sco.SiacoinOutput.Address), dbDecode(&sco.SiacoinOutput.Value)); err != nil { + if err := rows.Scan(&txnID, decode(&sco.StateElement.ID), decode(&sco.LeafIndex), decodeSlice(&sco.MerkleProof), &sco.Source, &sco.MaturityHeight, decode(&sco.SiacoinOutput.Address), decode(&sco.SiacoinOutput.Value)); err != nil { return nil, fmt.Errorf("failed to scan siacoin output: %w", err) } result[txnID] = append(result[txnID], sco) @@ -73,7 +73,7 @@ ORDER BY transaction_order ASC` for rows.Next() { var txnID int64 var sci types.SiacoinInput - if err := rows.Scan(&txnID, dbDecode(&sci.ParentID), dbDecode(&sci.UnlockConditions)); err != nil { + if err := rows.Scan(&txnID, decode(&sci.ParentID), decode(&sci.UnlockConditions)); err != nil { return nil, fmt.Errorf("failed to scan siacoin input: %w", err) } result[txnID] = append(result[txnID], sci) @@ -97,7 +97,7 @@ ORDER BY transaction_order ASC` for rows.Next() { var txnID int64 var sfi types.SiafundInput - if err := rows.Scan(&txnID, dbDecode(&sfi.ParentID), dbDecode(&sfi.UnlockConditions), dbDecode(&sfi.ClaimAddress)); err != nil { + if err := rows.Scan(&txnID, decode(&sfi.ParentID), decode(&sfi.UnlockConditions), decode(&sfi.ClaimAddress)); err != nil { return nil, fmt.Errorf("failed to scan siafund input: %w", err) } result[txnID] = append(result[txnID], sfi) @@ -123,7 +123,7 @@ ORDER BY ts.transaction_order ASC` for rows.Next() { var txnID int64 var sfo explorer.SiafundOutput - if err := rows.Scan(&txnID, dbDecode(&sfo.StateElement.ID), dbDecode(&sfo.StateElement.LeafIndex), dbDecode(&sfo.StateElement.MerkleProof), dbDecode(&sfo.ClaimStart), dbDecode(&sfo.SiafundOutput.Address), dbDecode(&sfo.SiafundOutput.Value)); err != nil { + if err := rows.Scan(&txnID, decode(&sfo.StateElement.ID), decode(&sfo.StateElement.LeafIndex), decodeSlice(&sfo.StateElement.MerkleProof), decode(&sfo.ClaimStart), decode(&sfo.SiafundOutput.Address), decode(&sfo.SiafundOutput.Value)); err != nil { return nil, fmt.Errorf("failed to scan siafund output: %w", err) } result[txnID] = append(result[txnID], sfo) @@ -152,7 +152,7 @@ ORDER BY contract_order` for validRows.Next() { var contractID int64 var sco types.SiacoinOutput - if err := validRows.Scan(&contractID, dbDecode(&sco.Address), dbDecode(&sco.Value)); err != nil { + if err := validRows.Scan(&contractID, decode(&sco.Address), decode(&sco.Value)); err != nil { return nil, fmt.Errorf("failed to scan valid proof output: %w", err) } @@ -174,7 +174,7 @@ ORDER BY contract_order` for missedRows.Next() { var contractID int64 var sco types.SiacoinOutput - if err := missedRows.Scan(&contractID, dbDecode(&sco.Address), dbDecode(&sco.Value)); err != nil { + if err := missedRows.Scan(&contractID, decode(&sco.Address), decode(&sco.Value)); err != nil { return nil, fmt.Errorf("failed to scan missed proof output: %w", err) } @@ -212,7 +212,7 @@ ORDER BY ts.transaction_order ASC` for rows.Next() { var txnID, contractID int64 var fc explorer.FileContract - if err := rows.Scan(&txnID, &contractID, dbDecode(&fc.StateElement.ID), dbDecode(&fc.StateElement.LeafIndex), dbDecode(&fc.StateElement.MerkleProof), &fc.Resolved, &fc.Valid, &fc.Filesize, dbDecode(&fc.FileMerkleRoot), &fc.WindowStart, &fc.WindowEnd, dbDecode(&fc.Payout), dbDecode(&fc.UnlockHash), &fc.RevisionNumber); err != nil { + if err := rows.Scan(&txnID, &contractID, decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), decodeSlice(&fc.StateElement.MerkleProof), &fc.Resolved, &fc.Valid, &fc.Filesize, decode(&fc.FileMerkleRoot), &fc.WindowStart, &fc.WindowEnd, decode(&fc.Payout), decode(&fc.UnlockHash), &fc.RevisionNumber); err != nil { return nil, fmt.Errorf("failed to scan file contract: %w", err) } @@ -255,7 +255,7 @@ ORDER BY ts.transaction_order ASC` for rows.Next() { var txnID, contractID int64 var fc explorer.FileContractRevision - if err := rows.Scan(&txnID, &contractID, dbDecode(&fc.ParentID), dbDecode(&fc.UnlockConditions), dbDecode(&fc.StateElement.ID), dbDecode(&fc.StateElement.LeafIndex), dbDecode(&fc.StateElement.MerkleProof), &fc.Resolved, &fc.Valid, &fc.Filesize, dbDecode(&fc.FileMerkleRoot), &fc.WindowStart, &fc.WindowEnd, dbDecode(&fc.Payout), dbDecode(&fc.UnlockHash), &fc.RevisionNumber); err != nil { + if err := rows.Scan(&txnID, &contractID, decode(&fc.ParentID), decode(&fc.UnlockConditions), decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), decodeSlice(&fc.StateElement.MerkleProof), &fc.Resolved, &fc.Valid, &fc.Filesize, decode(&fc.FileMerkleRoot), &fc.WindowStart, &fc.WindowEnd, decode(&fc.Payout), decode(&fc.UnlockHash), &fc.RevisionNumber); err != nil { return nil, fmt.Errorf("failed to scan file contract: %w", err) } @@ -280,7 +280,7 @@ ORDER BY ts.transaction_order ASC` // blockTransactionIDs returns the database ID for each transaction in the // block. func blockTransactionIDs(tx *txn, blockID types.BlockID) (dbIDs []int64, err error) { - rows, err := tx.Query(`SELECT transaction_id FROM block_transactions WHERE block_id = ? ORDER BY block_order`, dbEncode(blockID)) + rows, err := tx.Query(`SELECT transaction_id FROM block_transactions WHERE block_id = ? ORDER BY block_order`, encode(blockID)) if err != nil { return nil, err } @@ -303,7 +303,7 @@ FROM siacoin_elements sc INNER JOIN miner_payouts mp ON (mp.output_id = sc.id) WHERE mp.block_id = ? ORDER BY mp.block_order ASC` - rows, err := tx.Query(query, dbEncode(blockID)) + rows, err := tx.Query(query, encode(blockID)) if err != nil { return nil, fmt.Errorf("failed to query miner payout ids: %w", err) } @@ -312,7 +312,7 @@ ORDER BY mp.block_order ASC` var result []explorer.SiacoinOutput for rows.Next() { var output explorer.SiacoinOutput - if err := rows.Scan(dbDecode(&output.StateElement.ID), dbDecode(&output.StateElement.LeafIndex), dbDecode(&output.StateElement.MerkleProof), &output.Source, &output.MaturityHeight, dbDecode(&output.SiacoinOutput.Address), dbDecode(&output.SiacoinOutput.Value)); err != nil { + if err := rows.Scan(decode(&output.StateElement.ID), decode(&output.StateElement.LeafIndex), decodeSlice(&output.StateElement.MerkleProof), &output.Source, &output.MaturityHeight, decode(&output.SiacoinOutput.Address), decode(&output.SiacoinOutput.Value)); err != nil { return nil, fmt.Errorf("failed to scan miner payout: %w", err) } result = append(result, output) @@ -325,7 +325,7 @@ func transactionDatabaseIDs(tx *txn, txnIDs []types.TransactionID) (dbIDs []int6 encodedIDs := func(ids []types.TransactionID) []any { result := make([]any, len(ids)) for i, id := range ids { - result[i] = dbEncode(id) + result[i] = encode(id) } return result } From 54c843f988ca296e67bcf01782baa900d3ca7fd8 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Fri, 19 Apr 2024 14:01:53 -0400 Subject: [PATCH 05/19] add update.go --- explorer/update.go | 220 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 explorer/update.go diff --git a/explorer/update.go b/explorer/update.go new file mode 100644 index 00000000..dc0d9ffb --- /dev/null +++ b/explorer/update.go @@ -0,0 +1,220 @@ +package explorer + +import ( + "fmt" + + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" +) + +type ConsensusUpdate interface { + ForEachSiacoinElement(fn func(sce types.SiacoinElement, spent bool)) + ForEachSiafundElement(fn func(sfe types.SiafundElement, spent bool)) + ForEachFileContractElement(fn func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool)) +} + +type DBFileContract struct { + ID types.FileContractID + RevisionNumber uint64 +} + +type UpdateTx interface { + UpdateMaturedBalances(revert bool, height uint64) error + AddSiacoinElements(bid types.BlockID, update ConsensusUpdate, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) + AddSiafundElements(bid types.BlockID, spentElements, newElements []types.SiafundElement) (map[types.SiafundOutputID]int64, error) + AddFileContractElements(bid types.BlockID, update ConsensusUpdate) (map[DBFileContract]int64, error) + UpdateBalances(height uint64, spentSiacoinElements, newSiacoinElements []types.SiacoinElement, spentSiafundElements, newSiafundElements []types.SiafundElement) error + AddBlock(b types.Block, height uint64) error + AddMinerPayouts(bid types.BlockID, height uint64, scos []types.SiacoinOutput, dbIDs map[types.SiacoinOutputID]int64) error + AddTransactions(bid types.BlockID, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[DBFileContract]int64) error + + DeleteBlock(bid types.BlockID) error +} + +// applyChainUpdate atomically applies a chain update to a store +func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { + if err := tx.AddBlock(cau.Block, cau.State.Index.Height); err != nil { + return fmt.Errorf("applyUpdates: failed to add block: %w", err) + } else if err := tx.UpdateMaturedBalances(false, cau.State.Index.Height); err != nil { + return fmt.Errorf("applyUpdates: failed to update matured balances: %w", err) + } + + created := make(map[types.Hash256]bool) + ephemeral := make(map[types.Hash256]bool) + for _, txn := range cau.Block.Transactions { + for i := range txn.SiacoinOutputs { + created[types.Hash256(txn.SiacoinOutputID(i))] = true + } + for _, input := range txn.SiacoinInputs { + ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] + } + for i := range txn.SiafundOutputs { + created[types.Hash256(txn.SiafundOutputID(i))] = true + } + for _, input := range txn.SiafundInputs { + ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] + } + } + + // add new siacoin elements to the store + var newSiacoinElements, spentSiacoinElements []types.SiacoinElement + var ephemeralSiacoinElements []types.SiacoinElement + cau.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { + if ephemeral[se.ID] { + ephemeralSiacoinElements = append(ephemeralSiacoinElements, se) + return + } + + if spent { + spentSiacoinElements = append(spentSiacoinElements, se) + } else { + newSiacoinElements = append(newSiacoinElements, se) + } + }) + + var newSiafundElements, spentSiafundElements []types.SiafundElement + var ephemeralSiafundElements []types.SiafundElement + cau.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { + if ephemeral[se.ID] { + ephemeralSiafundElements = append(ephemeralSiafundElements, se) + return + } + + if spent { + spentSiafundElements = append(spentSiafundElements, se) + } else { + newSiafundElements = append(newSiafundElements, se) + } + }) + + scDBIds, err := tx.AddSiacoinElements( + cau.Block.ID(), + cau, + append(spentSiacoinElements, ephemeralSiacoinElements...), + newSiacoinElements, + ) + if err != nil { + return fmt.Errorf("applyUpdates: failed to add siacoin outputs: %w", err) + } + sfDBIds, err := tx.AddSiafundElements( + cau.Block.ID(), + append(spentSiafundElements, ephemeralSiafundElements...), + newSiafundElements, + ) + if err != nil { + return fmt.Errorf("applyUpdates: failed to add siafund outputs: %w", err) + } + if err := tx.UpdateBalances(cau.State.Index.Height, spentSiacoinElements, newSiacoinElements, spentSiafundElements, newSiafundElements); err != nil { + return fmt.Errorf("applyUpdates: failed to update balances: %w", err) + } + + fcDBIds, err := tx.AddFileContractElements(cau.Block.ID(), cau) + if err != nil { + return fmt.Errorf("applyUpdates: failed to add file contracts: %w", err) + } + + if err := tx.AddMinerPayouts(cau.Block.ID(), cau.State.Index.Height, cau.Block.MinerPayouts, scDBIds); err != nil { + return fmt.Errorf("applyUpdates: failed to add miner payouts: %w", err) + } else if err := tx.AddTransactions(cau.Block.ID(), cau.Block.Transactions, scDBIds, sfDBIds, fcDBIds); err != nil { + return fmt.Errorf("applyUpdates: failed to add transactions: addTransactions: %w", err) + } + return nil +} + +// revertChainUpdate atomically reverts a chain update from a store +func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types.ChainIndex) error { + if err := tx.UpdateMaturedBalances(true, revertedIndex.Height); err != nil { + return fmt.Errorf("revertUpdate: failed to update matured balances: %w", err) + } + + created := make(map[types.Hash256]bool) + ephemeral := make(map[types.Hash256]bool) + for _, txn := range cru.Block.Transactions { + for i := range txn.SiacoinOutputs { + created[types.Hash256(txn.SiacoinOutputID(i))] = true + } + for _, input := range txn.SiacoinInputs { + ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] + } + for i := range txn.SiafundOutputs { + created[types.Hash256(txn.SiafundOutputID(i))] = true + } + for _, input := range txn.SiafundInputs { + ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] + } + } + + // add new siacoin elements to the store + var newSiacoinElements, spentSiacoinElements []types.SiacoinElement + var ephemeralSiacoinElements []types.SiacoinElement + cru.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { + if ephemeral[se.ID] { + ephemeralSiacoinElements = append(ephemeralSiacoinElements, se) + return + } + + if spent { + newSiacoinElements = append(newSiacoinElements, se) + } else { + spentSiacoinElements = append(spentSiacoinElements, se) + } + }) + + var newSiafundElements, spentSiafundElements []types.SiafundElement + var ephemeralSiafundElements []types.SiafundElement + cru.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { + if ephemeral[se.ID] { + ephemeralSiafundElements = append(ephemeralSiafundElements, se) + return + } + + if spent { + newSiafundElements = append(newSiafundElements, se) + } else { + spentSiafundElements = append(spentSiafundElements, se) + } + }) + + // log.Println("REVERT!") + if _, err := tx.AddSiacoinElements( + cru.Block.ID(), + cru, + spentSiacoinElements, + append(newSiacoinElements, ephemeralSiacoinElements...), + ); err != nil { + return fmt.Errorf("revertUpdate: failed to update siacoin output state: %w", err) + } else if _, err := tx.AddSiafundElements( + cru.Block.ID(), + spentSiafundElements, + append(newSiafundElements, ephemeralSiafundElements...), + ); err != nil { + return fmt.Errorf("revertUpdate: failed to update siafund output state: %w", err) + } else if err := tx.UpdateBalances(revertedIndex.Height, spentSiacoinElements, newSiacoinElements, spentSiafundElements, newSiafundElements); err != nil { + return fmt.Errorf("revertUpdate: failed to update balances: %w", err) + } else if _, err := tx.AddFileContractElements(cru.Block.ID(), cru); err != nil { + return fmt.Errorf("revertUpdate: failed to update file contract state: %w", err) + } else if err := tx.DeleteBlock(cru.Block.ID()); err != nil { + return fmt.Errorf("revertUpdate: failed to delete block: %w", err) + } + return nil +} + +// UpdateChainState applies the reverts and updates. +func UpdateChainState(tx UpdateTx, crus []chain.RevertUpdate, caus []chain.ApplyUpdate) error { + for _, cru := range crus { + revertedIndex := types.ChainIndex{ + ID: cru.Block.ID(), + Height: cru.State.Index.Height + 1, + } + if err := revertChainUpdate(tx, cru, revertedIndex); err != nil { + return fmt.Errorf("failed to revert chain update %q: %w", revertedIndex, err) + } + } + + for _, cau := range caus { + if err := applyChainUpdate(tx, cau); err != nil { + return fmt.Errorf("failed to apply chain update %q: %w", cau.State.Index, err) + } + } + return nil +} From e697129aa06972542cd4e88e89ceb4cfae19c211 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Fri, 19 Apr 2024 14:24:25 -0400 Subject: [PATCH 06/19] linter fixes --- explorer/update.go | 7 ++++++- persist/sqlite/consensus.go | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/explorer/update.go b/explorer/update.go index dc0d9ffb..4abaa17b 100644 --- a/explorer/update.go +++ b/explorer/update.go @@ -7,23 +7,28 @@ import ( "go.sia.tech/coreutils/chain" ) +// A ConsensusUpdate is a chain apply or revert update. type ConsensusUpdate interface { ForEachSiacoinElement(fn func(sce types.SiacoinElement, spent bool)) ForEachSiafundElement(fn func(sfe types.SiafundElement, spent bool)) ForEachFileContractElement(fn func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool)) } +// A DBFileContract represents a file contract element in the DB. type DBFileContract struct { ID types.FileContractID RevisionNumber uint64 } +// An UpdateTx atomically updates the state of a store. type UpdateTx interface { - UpdateMaturedBalances(revert bool, height uint64) error AddSiacoinElements(bid types.BlockID, update ConsensusUpdate, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) AddSiafundElements(bid types.BlockID, spentElements, newElements []types.SiafundElement) (map[types.SiafundOutputID]int64, error) AddFileContractElements(bid types.BlockID, update ConsensusUpdate) (map[DBFileContract]int64, error) + UpdateBalances(height uint64, spentSiacoinElements, newSiacoinElements []types.SiacoinElement, spentSiafundElements, newSiafundElements []types.SiafundElement) error + UpdateMaturedBalances(revert bool, height uint64) error + AddBlock(b types.Block, height uint64) error AddMinerPayouts(bid types.BlockID, height uint64, scos []types.SiacoinOutput, dbIDs map[types.SiacoinOutputID]int64) error AddTransactions(bid types.BlockID, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[DBFileContract]int64) error diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index d982c28c..fc148203 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -147,7 +147,7 @@ func (ut *updateTx) AddFileContracts(id int64, txn types.Transaction, fcDBIds ma defer missedOutputsStmt.Close() for i := range txn.FileContracts { - dbID, ok := fcDBIds[explorer.DBFileContract{txn.FileContractID(i), 0}] + dbID, ok := fcDBIds[explorer.DBFileContract{ID: txn.FileContractID(i), RevisionNumber: 0}] if !ok { return errors.New("addFileContracts: fcDbID not in map") } @@ -192,7 +192,7 @@ func (ut *updateTx) AddFileContractRevisions(id int64, txn types.Transaction, db for i := range txn.FileContractRevisions { fcr := &txn.FileContractRevisions[i] - dbID, ok := dbIDs[explorer.DBFileContract{fcr.ParentID, fcr.FileContract.RevisionNumber}] + dbID, ok := dbIDs[explorer.DBFileContract{ID: fcr.ParentID, RevisionNumber: fcr.FileContract.RevisionNumber}] if !ok { return errors.New("addFileContractRevisions: dbID not in map") } @@ -581,7 +581,7 @@ func (ut *updateTx) AddFileContractElements(bid types.BlockID, update explorer.C return } - fcDBIds[explorer.DBFileContract{types.FileContractID(fce.StateElement.ID), fc.RevisionNumber}] = dbID + fcDBIds[explorer.DBFileContract{ID: types.FileContractID(fce.StateElement.ID), RevisionNumber: fc.RevisionNumber}] = dbID }) return fcDBIds, updateErr } From 7fc2d1aa37da0a5fe410f07b1a37449e0ab374e5 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Mon, 22 Apr 2024 14:40:18 -0400 Subject: [PATCH 07/19] use ForEachTreeNode to get state tree updates --- explorer/explorer.go | 6 +++ explorer/update.go | 80 ++++++++++++++++++++++++++----------- persist/sqlite/consensus.go | 16 ++++++++ persist/sqlite/init.go | 3 -- persist/sqlite/init.sql | 10 ++--- persist/sqlite/store.go | 2 - 6 files changed, 84 insertions(+), 33 deletions(-) diff --git a/explorer/explorer.go b/explorer/explorer.go index 1055f8c9..f6d69d84 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -34,6 +34,7 @@ type Store interface { Tip() (types.ChainIndex, error) Block(id types.BlockID) (Block, error) BestTip(height uint64) (types.ChainIndex, error) + MerkleProof(leafIndex uint64) ([]types.Hash256, error) Transactions(ids []types.TransactionID) ([]Transaction, error) UnspentSiacoinOutputs(address types.Address, limit, offset uint64) ([]SiacoinOutput, error) UnspentSiafundOutputs(address types.Address, limit, offset uint64) ([]SiafundOutput, error) @@ -124,6 +125,11 @@ func (e *Explorer) BestTip(height uint64) (types.ChainIndex, error) { return e.s.BestTip(height) } +// MerkleProof returns the proof of a given leaf. +func (e *Explorer) MerkleProof(leafIndex uint64) ([]types.Hash256, error) { + return e.s.MerkleProof(leafIndex) +} + // Transactions returns the transactions with the specified IDs. func (e *Explorer) Transactions(ids []types.TransactionID) ([]Transaction, error) { return e.s.Transactions(ids) diff --git a/explorer/update.go b/explorer/update.go index 4abaa17b..183abf90 100644 --- a/explorer/update.go +++ b/explorer/update.go @@ -7,34 +7,45 @@ import ( "go.sia.tech/coreutils/chain" ) -// A ConsensusUpdate is a chain apply or revert update. -type ConsensusUpdate interface { - ForEachSiacoinElement(fn func(sce types.SiacoinElement, spent bool)) - ForEachSiafundElement(fn func(sfe types.SiafundElement, spent bool)) - ForEachFileContractElement(fn func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool)) -} +type ( -// A DBFileContract represents a file contract element in the DB. -type DBFileContract struct { - ID types.FileContractID - RevisionNumber uint64 -} + // A ConsensusUpdate is a chain apply or revert update. + ConsensusUpdate interface { + ForEachSiacoinElement(fn func(sce types.SiacoinElement, spent bool)) + ForEachSiafundElement(fn func(sfe types.SiafundElement, spent bool)) + ForEachFileContractElement(fn func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool)) + } + + // A DBFileContract represents a file contract element in the DB. + DBFileContract struct { + ID types.FileContractID + RevisionNumber uint64 + } -// An UpdateTx atomically updates the state of a store. -type UpdateTx interface { - AddSiacoinElements(bid types.BlockID, update ConsensusUpdate, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) - AddSiafundElements(bid types.BlockID, spentElements, newElements []types.SiafundElement) (map[types.SiafundOutputID]int64, error) - AddFileContractElements(bid types.BlockID, update ConsensusUpdate) (map[DBFileContract]int64, error) + // A TreeNodeUpdate is a change to a merkle tree node. + TreeNodeUpdate struct { + Row uint64 + Column uint64 + Hash types.Hash256 + } - UpdateBalances(height uint64, spentSiacoinElements, newSiacoinElements []types.SiacoinElement, spentSiafundElements, newSiafundElements []types.SiafundElement) error - UpdateMaturedBalances(revert bool, height uint64) error + // An UpdateTx atomically updates the state of a store. + UpdateTx interface { + UpdateStateTree(changes []TreeNodeUpdate) error + AddSiacoinElements(bid types.BlockID, update ConsensusUpdate, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) + AddSiafundElements(bid types.BlockID, spentElements, newElements []types.SiafundElement) (map[types.SiafundOutputID]int64, error) + AddFileContractElements(bid types.BlockID, update ConsensusUpdate) (map[DBFileContract]int64, error) - AddBlock(b types.Block, height uint64) error - AddMinerPayouts(bid types.BlockID, height uint64, scos []types.SiacoinOutput, dbIDs map[types.SiacoinOutputID]int64) error - AddTransactions(bid types.BlockID, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[DBFileContract]int64) error + UpdateBalances(height uint64, spentSiacoinElements, newSiacoinElements []types.SiacoinElement, spentSiafundElements, newSiafundElements []types.SiafundElement) error + UpdateMaturedBalances(revert bool, height uint64) error - DeleteBlock(bid types.BlockID) error -} + AddBlock(b types.Block, height uint64) error + AddMinerPayouts(bid types.BlockID, height uint64, scos []types.SiacoinOutput, dbIDs map[types.SiacoinOutputID]int64) error + AddTransactions(bid types.BlockID, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[DBFileContract]int64) error + + DeleteBlock(bid types.BlockID) error + } +) // applyChainUpdate atomically applies a chain update to a store func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { @@ -92,6 +103,15 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { } }) + var treeUpdates []TreeNodeUpdate + cau.ForEachTreeNode(func(row, column uint64, hash types.Hash256) { + treeUpdates = append(treeUpdates, TreeNodeUpdate{ + Row: row, + Column: column, + Hash: hash, + }) + }) + scDBIds, err := tx.AddSiacoinElements( cau.Block.ID(), cau, @@ -122,7 +142,10 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { return fmt.Errorf("applyUpdates: failed to add miner payouts: %w", err) } else if err := tx.AddTransactions(cau.Block.ID(), cau.Block.Transactions, scDBIds, sfDBIds, fcDBIds); err != nil { return fmt.Errorf("applyUpdates: failed to add transactions: addTransactions: %w", err) + } else if err := tx.UpdateStateTree(treeUpdates); err != nil { + return fmt.Errorf("applyUpdates: failed to update state tree: %w", err) } + return nil } @@ -180,6 +203,15 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. } }) + var treeUpdates []TreeNodeUpdate + cru.ForEachTreeNode(func(row, column uint64, hash types.Hash256) { + treeUpdates = append(treeUpdates, TreeNodeUpdate{ + Row: row, + Column: column, + Hash: hash, + }) + }) + // log.Println("REVERT!") if _, err := tx.AddSiacoinElements( cru.Block.ID(), @@ -200,6 +232,8 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. return fmt.Errorf("revertUpdate: failed to update file contract state: %w", err) } else if err := tx.DeleteBlock(cru.Block.ID()); err != nil { return fmt.Errorf("revertUpdate: failed to delete block: %w", err) + } else if err := tx.UpdateStateTree(treeUpdates); err != nil { + return fmt.Errorf("revertUpdate: failed to update state tree: %w", err) } return nil } diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index fc148203..37a07b4e 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -432,6 +432,22 @@ func (ut *updateTx) UpdateMaturedBalances(revert bool, height uint64) error { return nil } +func (ut *updateTx) UpdateStateTree(changes []explorer.TreeNodeUpdate) error { + stmt, err := ut.tx.Prepare(`INSERT INTO state_tree (row, column, value) VALUES($1, $2, $3) ON CONFLICT (row, column) DO UPDATE SET value=EXCLUDED.value;`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + for _, change := range changes { + _, err := stmt.Exec(change.Row, change.Column, encode(change.Hash)) + if err != nil { + return fmt.Errorf("failed to execute statement: %w", err) + } + } + return nil +} + func (ut *updateTx) AddSiacoinElements(bid types.BlockID, update explorer.ConsensusUpdate, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) { sources := make(map[types.SiacoinOutputID]explorer.Source) if applyUpdate, ok := update.(chain.ApplyUpdate); ok { diff --git a/persist/sqlite/init.go b/persist/sqlite/init.go index dbf4ee3c..4d02da99 100644 --- a/persist/sqlite/init.go +++ b/persist/sqlite/init.go @@ -65,9 +65,6 @@ func (s *Store) init() error { // calculate the expected final database version target := int64(len(migrations) + 1) - // error is ignored -- the database may not have been initialized yet. - s.db.QueryRow("SELECT COUNT(*) FROM merkle_proofs WHERE i = 0").Scan(&s.numLeaves) - version := getDBVersion(s.db) switch { case version == 0: diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 79f092ee..ff611843 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -196,11 +196,11 @@ CREATE TABLE transaction_file_contract_revisions ( CREATE INDEX transaction_file_contract_revisions_transaction_id_index ON transaction_file_contract_revisions(transaction_id); -CREATE TABLE merkle_proofs ( - i INTEGER NOT NULL, - j INTEGER NOT NULL, - hash BLOB NOT NULL, - PRIMARY KEY(i ,j) +CREATE TABLE state_tree ( + row INTEGER NOT NULL, + column INTEGER NOT NULL, + value BLOB NOT NULL, + PRIMARY KEY(row, column) ); -- initialize the global settings table diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index c0d29512..7c03a7c2 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -19,8 +19,6 @@ type ( Store struct { db *sql.DB log *zap.Logger - - numLeaves uint64 } ) From 094bcd50f6ce8ae957a274e11d642240c3b7dfb9 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Mon, 22 Apr 2024 14:45:55 -0400 Subject: [PATCH 08/19] don't store merkle proofs inside element tables --- persist/sqlite/consensus.go | 28 ++++++++++++++-------------- persist/sqlite/contracts.go | 4 ++-- persist/sqlite/init.sql | 3 --- persist/sqlite/outputs.go | 8 ++++---- persist/sqlite/transactions.go | 20 ++++++++++---------- 5 files changed, 30 insertions(+), 33 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 37a07b4e..2f4af6da 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -473,10 +473,10 @@ func (ut *updateTx) AddSiacoinElements(bid types.BlockID, update explorer.Consen } } - stmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements(output_id, block_id, leaf_index, merkle_proof, spent, source, maturity_height, address, value) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + stmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements(output_id, block_id, leaf_index, spent, source, maturity_height, address, value) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (output_id) - DO UPDATE SET spent = ?, leaf_index = ?, merkle_proof = ?`) + DO UPDATE SET spent = ?, leaf_index = ?`) if err != nil { return nil, fmt.Errorf("addSiacoinElements: failed to prepare siacoin_elements statement: %w", err) } @@ -484,7 +484,7 @@ func (ut *updateTx) AddSiacoinElements(bid types.BlockID, update explorer.Consen scDBIds := make(map[types.SiacoinOutputID]int64) for _, sce := range newElements { - result, err := stmt.Exec(encode(sce.StateElement.ID), encode(bid), encode(sce.StateElement.LeafIndex), encodeSlice(sce.StateElement.MerkleProof), false, int(sources[types.SiacoinOutputID(sce.StateElement.ID)]), sce.MaturityHeight, encode(sce.SiacoinOutput.Address), encode(sce.SiacoinOutput.Value), false, encode(sce.StateElement.LeafIndex), encodeSlice(sce.StateElement.MerkleProof)) + result, err := stmt.Exec(encode(sce.StateElement.ID), encode(bid), encode(sce.StateElement.LeafIndex), false, int(sources[types.SiacoinOutputID(sce.StateElement.ID)]), sce.MaturityHeight, encode(sce.SiacoinOutput.Address), encode(sce.SiacoinOutput.Value), false, encode(sce.StateElement.LeafIndex)) if err != nil { return nil, fmt.Errorf("addSiacoinElements: failed to execute siacoin_elements statement: %w", err) } @@ -497,7 +497,7 @@ func (ut *updateTx) AddSiacoinElements(bid types.BlockID, update explorer.Consen scDBIds[types.SiacoinOutputID(sce.StateElement.ID)] = dbID } for _, sce := range spentElements { - result, err := stmt.Exec(encode(sce.StateElement.ID), encode(bid), encode(sce.StateElement.LeafIndex), encodeSlice(sce.StateElement.MerkleProof), true, int(sources[types.SiacoinOutputID(sce.StateElement.ID)]), sce.MaturityHeight, encode(sce.SiacoinOutput.Address), encode(sce.SiacoinOutput.Value), true, encode(sce.StateElement.LeafIndex), encodeSlice(sce.StateElement.MerkleProof)) + result, err := stmt.Exec(encode(sce.StateElement.ID), encode(bid), encode(sce.StateElement.LeafIndex), true, int(sources[types.SiacoinOutputID(sce.StateElement.ID)]), sce.MaturityHeight, encode(sce.SiacoinOutput.Address), encode(sce.SiacoinOutput.Value), true, encode(sce.StateElement.LeafIndex)) if err != nil { return nil, fmt.Errorf("addSiacoinElements: failed to execute siacoin_elements statement: %w", err) } @@ -514,10 +514,10 @@ func (ut *updateTx) AddSiacoinElements(bid types.BlockID, update explorer.Consen } func (ut *updateTx) AddSiafundElements(bid types.BlockID, spentElements, newElements []types.SiafundElement) (map[types.SiafundOutputID]int64, error) { - stmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements(output_id, block_id, leaf_index, merkle_proof, spent, claim_start, address, value) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) + stmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements(output_id, block_id, leaf_index, spent, claim_start, address, value) + VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT - DO UPDATE SET spent = ?, leaf_index = ?, merkle_proof = ?`) + DO UPDATE SET spent = ?, leaf_index = ?`) if err != nil { return nil, fmt.Errorf("addSiafundElements: failed to prepare siafund_elements statement: %w", err) } @@ -525,7 +525,7 @@ func (ut *updateTx) AddSiafundElements(bid types.BlockID, spentElements, newElem sfDBIds := make(map[types.SiafundOutputID]int64) for _, sfe := range newElements { - result, err := stmt.Exec(encode(sfe.StateElement.ID), encode(bid), encode(sfe.StateElement.LeafIndex), encodeSlice(sfe.StateElement.MerkleProof), false, encode(sfe.ClaimStart), encode(sfe.SiafundOutput.Address), encode(sfe.SiafundOutput.Value), false, encode(sfe.StateElement.LeafIndex), encodeSlice(sfe.StateElement.MerkleProof)) + result, err := stmt.Exec(encode(sfe.StateElement.ID), encode(bid), encode(sfe.StateElement.LeafIndex), false, encode(sfe.ClaimStart), encode(sfe.SiafundOutput.Address), encode(sfe.SiafundOutput.Value), false, encode(sfe.StateElement.LeafIndex)) if err != nil { return nil, fmt.Errorf("addSiafundElements: failed to execute siafund_elements statement: %w", err) } @@ -538,7 +538,7 @@ func (ut *updateTx) AddSiafundElements(bid types.BlockID, spentElements, newElem sfDBIds[types.SiafundOutputID(sfe.StateElement.ID)] = dbID } for _, sfe := range spentElements { - result, err := stmt.Exec(encode(sfe.StateElement.ID), encode(bid), encode(sfe.StateElement.LeafIndex), encodeSlice(sfe.StateElement.MerkleProof), true, encode(sfe.ClaimStart), encode(sfe.SiafundOutput.Address), encode(sfe.SiafundOutput.Value), true, encode(sfe.StateElement.LeafIndex), encodeSlice(sfe.StateElement.MerkleProof)) + result, err := stmt.Exec(encode(sfe.StateElement.ID), encode(bid), encode(sfe.StateElement.LeafIndex), true, encode(sfe.ClaimStart), encode(sfe.SiafundOutput.Address), encode(sfe.SiafundOutput.Value), true, encode(sfe.StateElement.LeafIndex)) if err != nil { return nil, fmt.Errorf("addSiafundElements: failed to execute siafund_elements statement: %w", err) } @@ -555,10 +555,10 @@ func (ut *updateTx) AddSiafundElements(bid types.BlockID, spentElements, newElem } func (ut *updateTx) AddFileContractElements(bid types.BlockID, update explorer.ConsensusUpdate) (map[explorer.DBFileContract]int64, error) { - stmt, err := ut.tx.Prepare(`INSERT INTO file_contract_elements(block_id, contract_id, leaf_index, merkle_proof, resolved, valid, filesize, file_merkle_root, window_start, window_end, payout, unlock_hash, revision_number) - VALUES (?, ?, ?, ?, FALSE, TRUE, ?, ?, ?, ?, ?, ?, ?) + stmt, err := ut.tx.Prepare(`INSERT INTO file_contract_elements(block_id, contract_id, leaf_index, resolved, valid, filesize, file_merkle_root, window_start, window_end, payout, unlock_hash, revision_number) + VALUES (?, ?, ?, FALSE, TRUE, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (contract_id, revision_number) - DO UPDATE SET resolved = ?, valid = ?, leaf_index = ?, merkle_proof = ? + DO UPDATE SET resolved = ?, valid = ?, leaf_index = ? RETURNING id;`) if err != nil { return nil, fmt.Errorf("addFileContractElements: failed to prepare file_contract_elements statement: %w", err) @@ -586,7 +586,7 @@ func (ut *updateTx) AddFileContractElements(bid types.BlockID, update explorer.C } var dbID int64 - err := stmt.QueryRow(encode(bid), encode(fce.StateElement.ID), encode(fce.StateElement.LeafIndex), encodeSlice(fce.StateElement.MerkleProof), fc.Filesize, encode(fc.FileMerkleRoot), fc.WindowStart, fc.WindowEnd, encode(fc.Payout), encode(fc.UnlockHash), fc.RevisionNumber, resolved, valid, encode(fce.StateElement.LeafIndex), encodeSlice(fce.StateElement.MerkleProof)).Scan(&dbID) + err := stmt.QueryRow(encode(bid), encode(fce.StateElement.ID), encode(fce.StateElement.LeafIndex), fc.Filesize, encode(fc.FileMerkleRoot), fc.WindowStart, fc.WindowEnd, encode(fc.Payout), encode(fc.UnlockHash), fc.RevisionNumber, resolved, valid, encode(fce.StateElement.LeafIndex)).Scan(&dbID) if err != nil { updateErr = fmt.Errorf("addFileContractElements: failed to execute file_contract_elements statement: %w", err) return diff --git a/persist/sqlite/contracts.go b/persist/sqlite/contracts.go index 386bdd22..b17b3ead 100644 --- a/persist/sqlite/contracts.go +++ b/persist/sqlite/contracts.go @@ -18,7 +18,7 @@ func (s *Store) Contracts(ids []types.FileContractID) (result []explorer.FileCon } err = s.transaction(func(tx *txn) error { - query := `SELECT fc1.id, fc1.contract_id, fc1.leaf_index, fc1.merkle_proof, fc1.resolved, fc1.valid, fc1.filesize, fc1.file_merkle_root, fc1.window_start, fc1.window_end, fc1.payout, fc1.unlock_hash, fc1.revision_number + query := `SELECT fc1.id, fc1.contract_id, fc1.leaf_index, fc1.resolved, fc1.valid, fc1.filesize, fc1.file_merkle_root, fc1.window_start, fc1.window_end, fc1.payout, fc1.unlock_hash, fc1.revision_number FROM file_contract_elements fc1 INNER JOIN last_contract_revision rev ON (rev.contract_element_id = fc1.id) WHERE rev.contract_id IN (` + queryPlaceHolders(len(ids)) + `)` @@ -33,7 +33,7 @@ func (s *Store) Contracts(ids []types.FileContractID) (result []explorer.FileCon for rows.Next() { var contractID int64 var fc explorer.FileContract - if err := rows.Scan(&contractID, decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), decodeSlice(&fc.StateElement.MerkleProof), &fc.Resolved, &fc.Valid, &fc.Filesize, decode(&fc.FileMerkleRoot), &fc.WindowStart, &fc.WindowEnd, decode(&fc.Payout), decode(&fc.UnlockHash), &fc.RevisionNumber); err != nil { + if err := rows.Scan(&contractID, decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), &fc.Resolved, &fc.Valid, &fc.Filesize, decode(&fc.FileMerkleRoot), &fc.WindowStart, &fc.WindowEnd, decode(&fc.Payout), decode(&fc.UnlockHash), &fc.RevisionNumber); err != nil { return fmt.Errorf("failed to scan transaction: %w", err) } diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index ff611843..2f86df92 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -26,7 +26,6 @@ CREATE TABLE siacoin_elements ( output_id BLOB UNIQUE NOT NULL, leaf_index BLOB NOT NULL, - merkle_proof BLOB NOT NULL, spent INTEGER NOT NULL, source INTEGER NOT NULL, @@ -44,7 +43,6 @@ CREATE TABLE siafund_elements ( output_id BLOB UNIQUE NOT NULL, leaf_index BLOB NOT NULL, - merkle_proof BLOB NOT NULL, spent INTEGER NOT NULL, claim_start BLOB NOT NULL, @@ -61,7 +59,6 @@ CREATE TABLE file_contract_elements ( contract_id BLOB NOT NULL, leaf_index BLOB NOT NULL, - merkle_proof BLOB NOT NULL, resolved INTEGER NOT NULL, valid INTEGER NOT NULL, diff --git a/persist/sqlite/outputs.go b/persist/sqlite/outputs.go index a5c9e392..209153d6 100644 --- a/persist/sqlite/outputs.go +++ b/persist/sqlite/outputs.go @@ -11,7 +11,7 @@ import ( // UnspentSiacoinOutputs implements explorer.Store. func (s *Store) UnspentSiacoinOutputs(address types.Address, limit, offset uint64) (result []explorer.SiacoinOutput, err error) { err = s.transaction(func(tx *txn) error { - rows, err := tx.Query(`SELECT output_id, leaf_index, merkle_proof, source, maturity_height, address, value FROM siacoin_elements WHERE address = ? AND spent = 0 LIMIT ? OFFSET ?`, encode(address), limit, offset) + rows, err := tx.Query(`SELECT output_id, leaf_index, source, maturity_height, address, value FROM siacoin_elements WHERE address = ? AND spent = 0 LIMIT ? OFFSET ?`, encode(address), limit, offset) if err != nil { return fmt.Errorf("failed to query siacoin outputs: %w", err) } @@ -19,7 +19,7 @@ func (s *Store) UnspentSiacoinOutputs(address types.Address, limit, offset uint6 for rows.Next() { var sco explorer.SiacoinOutput - if err := rows.Scan(decode(&sco.StateElement.ID), decode(&sco.StateElement.LeafIndex), decodeSlice(&sco.StateElement.MerkleProof), &sco.Source, &sco.MaturityHeight, decode(&sco.SiacoinOutput.Address), decode(&sco.SiacoinOutput.Value)); err != nil { + if err := rows.Scan(decode(&sco.StateElement.ID), decode(&sco.StateElement.LeafIndex), &sco.Source, &sco.MaturityHeight, decode(&sco.SiacoinOutput.Address), decode(&sco.SiacoinOutput.Value)); err != nil { return fmt.Errorf("failed to scan siacoin output: %w", err) } result = append(result, sco) @@ -32,7 +32,7 @@ func (s *Store) UnspentSiacoinOutputs(address types.Address, limit, offset uint6 // UnspentSiafundOutputs implements explorer.Store. func (s *Store) UnspentSiafundOutputs(address types.Address, limit, offset uint64) (result []explorer.SiafundOutput, err error) { err = s.transaction(func(tx *txn) error { - rows, err := tx.Query(`SELECT output_id, leaf_index, merkle_proof, claim_start, address, value FROM siafund_elements WHERE address = ? AND spent = 0 LIMIT ? OFFSET ?`, encode(address), limit, offset) + rows, err := tx.Query(`SELECT output_id, leaf_index, claim_start, address, value FROM siafund_elements WHERE address = ? AND spent = 0 LIMIT ? OFFSET ?`, encode(address), limit, offset) if err != nil { return fmt.Errorf("failed to query siafund outputs: %w", err) } @@ -40,7 +40,7 @@ func (s *Store) UnspentSiafundOutputs(address types.Address, limit, offset uint6 for rows.Next() { var sfo explorer.SiafundOutput - if err := rows.Scan(decode(&sfo.StateElement.ID), decode(&sfo.StateElement.LeafIndex), decodeSlice(&sfo.StateElement.MerkleProof), decode(&sfo.ClaimStart), decode(&sfo.SiafundOutput.Address), decode(&sfo.SiafundOutput.Value)); err != nil { + if err := rows.Scan(decode(&sfo.StateElement.ID), decode(&sfo.StateElement.LeafIndex), decode(&sfo.ClaimStart), decode(&sfo.SiafundOutput.Address), decode(&sfo.SiafundOutput.Value)); err != nil { return fmt.Errorf("failed to scan siafund output: %w", err) } result = append(result, sfo) diff --git a/persist/sqlite/transactions.go b/persist/sqlite/transactions.go index a9f7cbca..b3040db9 100644 --- a/persist/sqlite/transactions.go +++ b/persist/sqlite/transactions.go @@ -33,7 +33,7 @@ ORDER BY transaction_order ASC` // transactionSiacoinOutputs returns the siacoin outputs for each transaction. func transactionSiacoinOutputs(tx *txn, txnIDs []int64) (map[int64][]explorer.SiacoinOutput, error) { - query := `SELECT ts.transaction_id, sc.output_id, sc.leaf_index, sc.merkle_proof, sc.source, sc.maturity_height, sc.address, sc.value + query := `SELECT ts.transaction_id, sc.output_id, sc.leaf_index, sc.source, sc.maturity_height, sc.address, sc.value FROM siacoin_elements sc INNER JOIN transaction_siacoin_outputs ts ON (ts.output_id = sc.id) WHERE ts.transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `) @@ -49,7 +49,7 @@ ORDER BY ts.transaction_order ASC` for rows.Next() { var txnID int64 var sco explorer.SiacoinOutput - if err := rows.Scan(&txnID, decode(&sco.StateElement.ID), decode(&sco.LeafIndex), decodeSlice(&sco.MerkleProof), &sco.Source, &sco.MaturityHeight, decode(&sco.SiacoinOutput.Address), decode(&sco.SiacoinOutput.Value)); err != nil { + if err := rows.Scan(&txnID, decode(&sco.StateElement.ID), decode(&sco.LeafIndex), &sco.Source, &sco.MaturityHeight, decode(&sco.SiacoinOutput.Address), decode(&sco.SiacoinOutput.Value)); err != nil { return nil, fmt.Errorf("failed to scan siacoin output: %w", err) } result[txnID] = append(result[txnID], sco) @@ -107,7 +107,7 @@ ORDER BY transaction_order ASC` // transactionSiafundOutputs returns the siafund outputs for each transaction. func transactionSiafundOutputs(tx *txn, txnIDs []int64) (map[int64][]explorer.SiafundOutput, error) { - query := `SELECT ts.transaction_id, sf.output_id, sf.leaf_index, sf.merkle_proof, sf.claim_start, sf.address, sf.value + query := `SELECT ts.transaction_id, sf.output_id, sf.leaf_index, sf.claim_start, sf.address, sf.value FROM siafund_elements sf INNER JOIN transaction_siafund_outputs ts ON (ts.output_id = sf.id) WHERE ts.transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `) @@ -123,7 +123,7 @@ ORDER BY ts.transaction_order ASC` for rows.Next() { var txnID int64 var sfo explorer.SiafundOutput - if err := rows.Scan(&txnID, decode(&sfo.StateElement.ID), decode(&sfo.StateElement.LeafIndex), decodeSlice(&sfo.StateElement.MerkleProof), decode(&sfo.ClaimStart), decode(&sfo.SiafundOutput.Address), decode(&sfo.SiafundOutput.Value)); err != nil { + if err := rows.Scan(&txnID, decode(&sfo.StateElement.ID), decode(&sfo.StateElement.LeafIndex), decode(&sfo.ClaimStart), decode(&sfo.SiafundOutput.Address), decode(&sfo.SiafundOutput.Value)); err != nil { return nil, fmt.Errorf("failed to scan siafund output: %w", err) } result[txnID] = append(result[txnID], sfo) @@ -193,7 +193,7 @@ type contractOrder struct { // transactionFileContracts returns the file contracts for each transaction. func transactionFileContracts(tx *txn, txnIDs []int64) (map[int64][]explorer.FileContract, error) { - query := `SELECT ts.transaction_id, fc.id, fc.contract_id, fc.leaf_index, fc.merkle_proof, fc.resolved, fc.valid, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number + query := `SELECT ts.transaction_id, fc.id, fc.contract_id, fc.leaf_index, fc.resolved, fc.valid, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number FROM file_contract_elements fc INNER JOIN transaction_file_contracts ts ON (ts.contract_id = fc.id) WHERE ts.transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `) @@ -212,7 +212,7 @@ ORDER BY ts.transaction_order ASC` for rows.Next() { var txnID, contractID int64 var fc explorer.FileContract - if err := rows.Scan(&txnID, &contractID, decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), decodeSlice(&fc.StateElement.MerkleProof), &fc.Resolved, &fc.Valid, &fc.Filesize, decode(&fc.FileMerkleRoot), &fc.WindowStart, &fc.WindowEnd, decode(&fc.Payout), decode(&fc.UnlockHash), &fc.RevisionNumber); err != nil { + if err := rows.Scan(&txnID, &contractID, decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), &fc.Resolved, &fc.Valid, &fc.Filesize, decode(&fc.FileMerkleRoot), &fc.WindowStart, &fc.WindowEnd, decode(&fc.Payout), decode(&fc.UnlockHash), &fc.RevisionNumber); err != nil { return nil, fmt.Errorf("failed to scan file contract: %w", err) } @@ -236,7 +236,7 @@ ORDER BY ts.transaction_order ASC` // transactionFileContracts returns the file contract revisions for each transaction. func transactionFileContractRevisions(tx *txn, txnIDs []int64) (map[int64][]explorer.FileContractRevision, error) { - query := `SELECT ts.transaction_id, fc.id, ts.parent_id, ts.unlock_conditions, fc.contract_id, fc.leaf_index, fc.merkle_proof, fc.resolved, fc.valid, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number + query := `SELECT ts.transaction_id, fc.id, ts.parent_id, ts.unlock_conditions, fc.contract_id, fc.leaf_index, fc.resolved, fc.valid, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number FROM file_contract_elements fc INNER JOIN transaction_file_contract_revisions ts ON (ts.contract_id = fc.id) WHERE ts.transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `) @@ -255,7 +255,7 @@ ORDER BY ts.transaction_order ASC` for rows.Next() { var txnID, contractID int64 var fc explorer.FileContractRevision - if err := rows.Scan(&txnID, &contractID, decode(&fc.ParentID), decode(&fc.UnlockConditions), decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), decodeSlice(&fc.StateElement.MerkleProof), &fc.Resolved, &fc.Valid, &fc.Filesize, decode(&fc.FileMerkleRoot), &fc.WindowStart, &fc.WindowEnd, decode(&fc.Payout), decode(&fc.UnlockHash), &fc.RevisionNumber); err != nil { + if err := rows.Scan(&txnID, &contractID, decode(&fc.ParentID), decode(&fc.UnlockConditions), decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), &fc.Resolved, &fc.Valid, &fc.Filesize, decode(&fc.FileMerkleRoot), &fc.WindowStart, &fc.WindowEnd, decode(&fc.Payout), decode(&fc.UnlockHash), &fc.RevisionNumber); err != nil { return nil, fmt.Errorf("failed to scan file contract: %w", err) } @@ -298,7 +298,7 @@ func blockTransactionIDs(tx *txn, blockID types.BlockID) (dbIDs []int64, err err // blockMinerPayouts returns the miner payouts for the block. func blockMinerPayouts(tx *txn, blockID types.BlockID) ([]explorer.SiacoinOutput, error) { - query := `SELECT sc.output_id, sc.leaf_index, sc.merkle_proof, sc.source, sc.maturity_height, sc.address, sc.value + query := `SELECT sc.output_id, sc.leaf_index, sc.source, sc.maturity_height, sc.address, sc.value FROM siacoin_elements sc INNER JOIN miner_payouts mp ON (mp.output_id = sc.id) WHERE mp.block_id = ? @@ -312,7 +312,7 @@ ORDER BY mp.block_order ASC` var result []explorer.SiacoinOutput for rows.Next() { var output explorer.SiacoinOutput - if err := rows.Scan(decode(&output.StateElement.ID), decode(&output.StateElement.LeafIndex), decodeSlice(&output.StateElement.MerkleProof), &output.Source, &output.MaturityHeight, decode(&output.SiacoinOutput.Address), decode(&output.SiacoinOutput.Value)); err != nil { + if err := rows.Scan(decode(&output.StateElement.ID), decode(&output.StateElement.LeafIndex), &output.Source, &output.MaturityHeight, decode(&output.SiacoinOutput.Address), decode(&output.SiacoinOutput.Value)); err != nil { return nil, fmt.Errorf("failed to scan miner payout: %w", err) } result = append(result, output) From 1ab1c62ea6e21f808d001dd31d784b2033613dc2 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Mon, 22 Apr 2024 15:05:26 -0400 Subject: [PATCH 09/19] add merkle.go --- persist/sqlite/merkle.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 persist/sqlite/merkle.go diff --git a/persist/sqlite/merkle.go b/persist/sqlite/merkle.go new file mode 100644 index 00000000..4e8f19ef --- /dev/null +++ b/persist/sqlite/merkle.go @@ -0,0 +1,39 @@ +package sqlite + +import ( + "math/bits" + + "go.sia.tech/core/types" +) + +// MerkleProof implements explorer.Store. +func (s *Store) MerkleProof(leafIndex uint64) (proof []types.Hash256, err error) { + err = s.transaction(func(tx *txn) error { + var numLeaves uint64 + if err := tx.QueryRow("SELECT COUNT(*) FROM state_tree WHERE i = 0").Scan(&numLeaves); err != nil { + return err + } + + pos := leafIndex + stmt, err := tx.Prepare("SELECT hash FROM state_tree WHERE row = ? AND column = ?") + if err != nil { + return err + } + + proof = make([]types.Hash256, bits.Len64(leafIndex^numLeaves)-1) + for i := range proof { + subtreeSize := uint64(1 << i) + if leafIndex&(1< Date: Tue, 23 Apr 2024 13:08:28 -0400 Subject: [PATCH 10/19] update code so we don't pass apply update/revert update to UpdateTx methods --- explorer/update.go | 64 ++++++++++++++++++++++++++++++------- persist/sqlite/consensus.go | 48 ++++++---------------------- 2 files changed, 62 insertions(+), 50 deletions(-) diff --git a/explorer/update.go b/explorer/update.go index 183abf90..95848e54 100644 --- a/explorer/update.go +++ b/explorer/update.go @@ -8,12 +8,11 @@ import ( ) type ( - - // A ConsensusUpdate is a chain apply or revert update. - ConsensusUpdate interface { - ForEachSiacoinElement(fn func(sce types.SiacoinElement, spent bool)) - ForEachSiafundElement(fn func(sfe types.SiafundElement, spent bool)) - ForEachFileContractElement(fn func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool)) + // FileContractUpdate represents a file contract from a consensus update. + FileContractUpdate struct { + FileContractElement types.FileContractElement + Revision *types.FileContractElement + Resolved, Valid bool } // A DBFileContract represents a file contract element in the DB. @@ -32,9 +31,9 @@ type ( // An UpdateTx atomically updates the state of a store. UpdateTx interface { UpdateStateTree(changes []TreeNodeUpdate) error - AddSiacoinElements(bid types.BlockID, update ConsensusUpdate, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) + AddSiacoinElements(bid types.BlockID, sources map[types.SiacoinOutputID]Source, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) AddSiafundElements(bid types.BlockID, spentElements, newElements []types.SiafundElement) (map[types.SiafundOutputID]int64, error) - AddFileContractElements(bid types.BlockID, update ConsensusUpdate) (map[DBFileContract]int64, error) + AddFileContractElements(bid types.BlockID, fces []FileContractUpdate) (map[DBFileContract]int64, error) UpdateBalances(height uint64, spentSiacoinElements, newSiacoinElements []types.SiacoinElement, spentSiafundElements, newSiafundElements []types.SiafundElement) error UpdateMaturedBalances(revert bool, height uint64) error @@ -55,6 +54,27 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { return fmt.Errorf("applyUpdates: failed to update matured balances: %w", err) } + sources := make(map[types.SiacoinOutputID]Source) + for i := range cau.Block.MinerPayouts { + sources[cau.Block.ID().MinerOutputID(i)] = SourceMinerPayout + } + + for _, txn := range cau.Block.Transactions { + for i := range txn.SiacoinOutputs { + sources[txn.SiacoinOutputID(i)] = SourceTransaction + } + + for i := range txn.FileContracts { + fcid := txn.FileContractID(i) + for j := range txn.FileContracts[i].ValidProofOutputs { + sources[fcid.ValidOutputID(j)] = SourceValidProofOutput + } + for j := range txn.FileContracts[i].MissedProofOutputs { + sources[fcid.MissedOutputID(j)] = SourceMissedProofOutput + } + } + } + created := make(map[types.Hash256]bool) ephemeral := make(map[types.Hash256]bool) for _, txn := range cau.Block.Transactions { @@ -103,6 +123,16 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { } }) + var fces []FileContractUpdate + cau.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { + fces = append(fces, FileContractUpdate{ + FileContractElement: fce, + Revision: rev, + Resolved: resolved, + Valid: valid, + }) + }) + var treeUpdates []TreeNodeUpdate cau.ForEachTreeNode(func(row, column uint64, hash types.Hash256) { treeUpdates = append(treeUpdates, TreeNodeUpdate{ @@ -114,7 +144,7 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { scDBIds, err := tx.AddSiacoinElements( cau.Block.ID(), - cau, + sources, append(spentSiacoinElements, ephemeralSiacoinElements...), newSiacoinElements, ) @@ -133,7 +163,7 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { return fmt.Errorf("applyUpdates: failed to update balances: %w", err) } - fcDBIds, err := tx.AddFileContractElements(cau.Block.ID(), cau) + fcDBIds, err := tx.AddFileContractElements(cau.Block.ID(), fces) if err != nil { return fmt.Errorf("applyUpdates: failed to add file contracts: %w", err) } @@ -203,6 +233,16 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. } }) + var fces []FileContractUpdate + cru.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { + fces = append(fces, FileContractUpdate{ + FileContractElement: fce, + Revision: rev, + Resolved: resolved, + Valid: valid, + }) + }) + var treeUpdates []TreeNodeUpdate cru.ForEachTreeNode(func(row, column uint64, hash types.Hash256) { treeUpdates = append(treeUpdates, TreeNodeUpdate{ @@ -215,7 +255,7 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. // log.Println("REVERT!") if _, err := tx.AddSiacoinElements( cru.Block.ID(), - cru, + nil, spentSiacoinElements, append(newSiacoinElements, ephemeralSiacoinElements...), ); err != nil { @@ -228,7 +268,7 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. return fmt.Errorf("revertUpdate: failed to update siafund output state: %w", err) } else if err := tx.UpdateBalances(revertedIndex.Height, spentSiacoinElements, newSiacoinElements, spentSiafundElements, newSiafundElements); err != nil { return fmt.Errorf("revertUpdate: failed to update balances: %w", err) - } else if _, err := tx.AddFileContractElements(cru.Block.ID(), cru); err != nil { + } else if _, err := tx.AddFileContractElements(cru.Block.ID(), fces); err != nil { return fmt.Errorf("revertUpdate: failed to update file contract state: %w", err) } else if err := tx.DeleteBlock(cru.Block.ID()); err != nil { return fmt.Errorf("revertUpdate: failed to delete block: %w", err) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 2f4af6da..afda4eb7 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -448,31 +448,7 @@ func (ut *updateTx) UpdateStateTree(changes []explorer.TreeNodeUpdate) error { return nil } -func (ut *updateTx) AddSiacoinElements(bid types.BlockID, update explorer.ConsensusUpdate, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) { - sources := make(map[types.SiacoinOutputID]explorer.Source) - if applyUpdate, ok := update.(chain.ApplyUpdate); ok { - block := applyUpdate.Block - for i := range block.MinerPayouts { - sources[bid.MinerOutputID(i)] = explorer.SourceMinerPayout - } - - for _, txn := range block.Transactions { - for i := range txn.SiacoinOutputs { - sources[txn.SiacoinOutputID(i)] = explorer.SourceTransaction - } - - for i := range txn.FileContracts { - fcid := txn.FileContractID(i) - for j := range txn.FileContracts[i].ValidProofOutputs { - sources[fcid.ValidOutputID(j)] = explorer.SourceValidProofOutput - } - for j := range txn.FileContracts[i].MissedProofOutputs { - sources[fcid.MissedOutputID(j)] = explorer.SourceMissedProofOutput - } - } - } - } - +func (ut *updateTx) AddSiacoinElements(bid types.BlockID, sources map[types.SiacoinOutputID]explorer.Source, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) { stmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements(output_id, block_id, leaf_index, spent, source, maturity_height, address, value) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (output_id) @@ -554,7 +530,7 @@ func (ut *updateTx) AddSiafundElements(bid types.BlockID, spentElements, newElem return sfDBIds, nil } -func (ut *updateTx) AddFileContractElements(bid types.BlockID, update explorer.ConsensusUpdate) (map[explorer.DBFileContract]int64, error) { +func (ut *updateTx) AddFileContractElements(bid types.BlockID, fces []explorer.FileContractUpdate) (map[explorer.DBFileContract]int64, error) { stmt, err := ut.tx.Prepare(`INSERT INTO file_contract_elements(block_id, contract_id, leaf_index, resolved, valid, filesize, file_merkle_root, window_start, window_end, payout, unlock_hash, revision_number) VALUES (?, ?, ?, FALSE, TRUE, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (contract_id, revision_number) @@ -575,30 +551,26 @@ func (ut *updateTx) AddFileContractElements(bid types.BlockID, update explorer.C var updateErr error fcDBIds := make(map[explorer.DBFileContract]int64) - update.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { - if updateErr != nil { - return - } + for _, update := range fces { + fce := update.FileContractElement fc := &fce.FileContract - if rev != nil { - fc = &rev.FileContract + if update.Revision != nil { + fc = &update.Revision.FileContract } var dbID int64 - err := stmt.QueryRow(encode(bid), encode(fce.StateElement.ID), encode(fce.StateElement.LeafIndex), fc.Filesize, encode(fc.FileMerkleRoot), fc.WindowStart, fc.WindowEnd, encode(fc.Payout), encode(fc.UnlockHash), fc.RevisionNumber, resolved, valid, encode(fce.StateElement.LeafIndex)).Scan(&dbID) + err := stmt.QueryRow(encode(bid), encode(fce.StateElement.ID), encode(fce.StateElement.LeafIndex), fc.Filesize, encode(fc.FileMerkleRoot), fc.WindowStart, fc.WindowEnd, encode(fc.Payout), encode(fc.UnlockHash), fc.RevisionNumber, update.Resolved, update.Valid, encode(fce.StateElement.LeafIndex)).Scan(&dbID) if err != nil { - updateErr = fmt.Errorf("addFileContractElements: failed to execute file_contract_elements statement: %w", err) - return + return nil, fmt.Errorf("addFileContractElements: failed to execute file_contract_elements statement: %w", err) } if _, err := revisionStmt.Exec(encode(fce.StateElement.ID), dbID, dbID); err != nil { - updateErr = fmt.Errorf("addFileContractElements: failed to update last revision number: %w", err) - return + return nil, fmt.Errorf("addFileContractElements: failed to update last revision number: %w", err) } fcDBIds[explorer.DBFileContract{ID: types.FileContractID(fce.StateElement.ID), RevisionNumber: fc.RevisionNumber}] = dbID - }) + } return fcDBIds, updateErr } From 7d054022c89cf2b33be5fca89db47d8b75ea6e44 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Wed, 24 Apr 2024 11:28:09 -0400 Subject: [PATCH 11/19] don't export methods if not necessary --- persist/sqlite/consensus.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index afda4eb7..f9530c9d 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -41,7 +41,7 @@ func (ut *updateTx) AddMinerPayouts(bid types.BlockID, height uint64, scos []typ return nil } -func (ut *updateTx) AddArbitraryData(id int64, txn types.Transaction) error { +func (ut *updateTx) addArbitraryData(id int64, txn types.Transaction) error { stmt, err := ut.tx.Prepare(`INSERT INTO transaction_arbitrary_data(transaction_id, transaction_order, data) VALUES (?, ?, ?)`) if err != nil { @@ -57,7 +57,7 @@ func (ut *updateTx) AddArbitraryData(id int64, txn types.Transaction) error { return nil } -func (ut *updateTx) AddSiacoinInputs(id int64, txn types.Transaction) error { +func (ut *updateTx) addSiacoinInputs(id int64, txn types.Transaction) error { stmt, err := ut.tx.Prepare(`INSERT INTO transaction_siacoin_inputs(transaction_id, transaction_order, parent_id, unlock_conditions) VALUES (?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addSiacoinInputs: failed to prepare statement: %w", err) @@ -72,7 +72,7 @@ func (ut *updateTx) AddSiacoinInputs(id int64, txn types.Transaction) error { return nil } -func (ut *updateTx) AddSiacoinOutputs(id int64, txn types.Transaction, dbIDs map[types.SiacoinOutputID]int64) error { +func (ut *updateTx) addSiacoinOutputs(id int64, txn types.Transaction, dbIDs map[types.SiacoinOutputID]int64) error { stmt, err := ut.tx.Prepare(`INSERT INTO transaction_siacoin_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, ?)`) if err != nil { return fmt.Errorf("addSiacoinOutputs: failed to prepare statement: %w", err) @@ -92,7 +92,7 @@ func (ut *updateTx) AddSiacoinOutputs(id int64, txn types.Transaction, dbIDs map return nil } -func (ut *updateTx) AddSiafundInputs(id int64, txn types.Transaction) error { +func (ut *updateTx) addSiafundInputs(id int64, txn types.Transaction) error { stmt, err := ut.tx.Prepare(`INSERT INTO transaction_siafund_inputs(transaction_id, transaction_order, parent_id, unlock_conditions, claim_address) VALUES (?, ?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addSiafundInputs: failed to prepare statement: %w", err) @@ -107,7 +107,7 @@ func (ut *updateTx) AddSiafundInputs(id int64, txn types.Transaction) error { return nil } -func (ut *updateTx) AddSiafundOutputs(id int64, txn types.Transaction, dbIDs map[types.SiafundOutputID]int64) error { +func (ut *updateTx) addSiafundOutputs(id int64, txn types.Transaction, dbIDs map[types.SiafundOutputID]int64) error { stmt, err := ut.tx.Prepare(`INSERT INTO transaction_siafund_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, ?)`) if err != nil { return fmt.Errorf("addSiafundOutputs: failed to prepare statement: %w", err) @@ -127,7 +127,7 @@ func (ut *updateTx) AddSiafundOutputs(id int64, txn types.Transaction, dbIDs map return nil } -func (ut *updateTx) AddFileContracts(id int64, txn types.Transaction, fcDBIds map[explorer.DBFileContract]int64) error { +func (ut *updateTx) addFileContracts(id int64, txn types.Transaction, fcDBIds map[explorer.DBFileContract]int64) error { stmt, err := ut.tx.Prepare(`INSERT INTO transaction_file_contracts(transaction_id, transaction_order, contract_id) VALUES (?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContracts: failed to prepare statement: %w", err) @@ -171,7 +171,7 @@ func (ut *updateTx) AddFileContracts(id int64, txn types.Transaction, fcDBIds ma return nil } -func (ut *updateTx) AddFileContractRevisions(id int64, txn types.Transaction, dbIDs map[explorer.DBFileContract]int64) error { +func (ut *updateTx) addFileContractRevisions(id int64, txn types.Transaction, dbIDs map[explorer.DBFileContract]int64) error { stmt, err := ut.tx.Prepare(`INSERT INTO transaction_file_contract_revisions(transaction_id, transaction_order, contract_id, parent_id, unlock_conditions) VALUES (?, ?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContractRevisions: failed to prepare statement: %w", err) @@ -241,19 +241,19 @@ func (ut *updateTx) AddTransactions(bid types.BlockID, txns []types.Transaction, if _, err := blockTransactionsStmt.Exec(encode(bid), txnID, i); err != nil { return fmt.Errorf("failed to insert into block_transactions: %w", err) - } else if err := ut.AddArbitraryData(txnID, txn); err != nil { + } else if err := ut.addArbitraryData(txnID, txn); err != nil { return fmt.Errorf("failed to add arbitrary data: %w", err) - } else if err := ut.AddSiacoinInputs(txnID, txn); err != nil { + } else if err := ut.addSiacoinInputs(txnID, txn); err != nil { return fmt.Errorf("failed to add siacoin inputs: %w", err) - } else if err := ut.AddSiacoinOutputs(txnID, txn, scDBIds); err != nil { + } else if err := ut.addSiacoinOutputs(txnID, txn, scDBIds); err != nil { return fmt.Errorf("failed to add siacoin outputs: %w", err) - } else if err := ut.AddSiafundInputs(txnID, txn); err != nil { + } else if err := ut.addSiafundInputs(txnID, txn); err != nil { return fmt.Errorf("failed to add siafund inputs: %w", err) - } else if err := ut.AddSiafundOutputs(txnID, txn, sfDBIds); err != nil { + } else if err := ut.addSiafundOutputs(txnID, txn, sfDBIds); err != nil { return fmt.Errorf("failed to add siafund outputs: %w", err) - } else if err := ut.AddFileContracts(txnID, txn, fcDBIds); err != nil { + } else if err := ut.addFileContracts(txnID, txn, fcDBIds); err != nil { return fmt.Errorf("failed to add file contract: %w", err) - } else if err := ut.AddFileContractRevisions(txnID, txn, fcDBIds); err != nil { + } else if err := ut.addFileContractRevisions(txnID, txn, fcDBIds); err != nil { return fmt.Errorf("failed to add file contract revisions: %w", err) } } From 6423133544c94b735a76ecc74da59ec8f5478859 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Thu, 25 Apr 2024 23:00:33 -0400 Subject: [PATCH 12/19] make UpdateTx interface more generic --- explorer/update.go | 128 ++++++++++++++---------------------- persist/sqlite/consensus.go | 79 +++++++++++++++++++++- 2 files changed, 126 insertions(+), 81 deletions(-) diff --git a/explorer/update.go b/explorer/update.go index 95848e54..948b2fa6 100644 --- a/explorer/update.go +++ b/explorer/update.go @@ -28,32 +28,34 @@ type ( Hash types.Hash256 } + // An UpdateState contains information relevant to the block being applied + // or reverted. + UpdateState struct { + Block types.Block + Index types.ChainIndex + TreeUpdates []TreeNodeUpdate + + Sources map[types.SiacoinOutputID]Source + NewSiacoinElements []types.SiacoinElement + SpentSiacoinElements []types.SiacoinElement + EphemeralSiacoinElements []types.SiacoinElement + + NewSiafundElements []types.SiafundElement + SpentSiafundElements []types.SiafundElement + EphemeralSiafundElements []types.SiafundElement + + FileContractElements []FileContractUpdate + } + // An UpdateTx atomically updates the state of a store. UpdateTx interface { - UpdateStateTree(changes []TreeNodeUpdate) error - AddSiacoinElements(bid types.BlockID, sources map[types.SiacoinOutputID]Source, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) - AddSiafundElements(bid types.BlockID, spentElements, newElements []types.SiafundElement) (map[types.SiafundOutputID]int64, error) - AddFileContractElements(bid types.BlockID, fces []FileContractUpdate) (map[DBFileContract]int64, error) - - UpdateBalances(height uint64, spentSiacoinElements, newSiacoinElements []types.SiacoinElement, spentSiafundElements, newSiafundElements []types.SiafundElement) error - UpdateMaturedBalances(revert bool, height uint64) error - - AddBlock(b types.Block, height uint64) error - AddMinerPayouts(bid types.BlockID, height uint64, scos []types.SiacoinOutput, dbIDs map[types.SiacoinOutputID]int64) error - AddTransactions(bid types.BlockID, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[DBFileContract]int64) error - - DeleteBlock(bid types.BlockID) error + ApplyIndex(state UpdateState) error + RevertIndex(state UpdateState) error } ) // applyChainUpdate atomically applies a chain update to a store func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { - if err := tx.AddBlock(cau.Block, cau.State.Index.Height); err != nil { - return fmt.Errorf("applyUpdates: failed to add block: %w", err) - } else if err := tx.UpdateMaturedBalances(false, cau.State.Index.Height); err != nil { - return fmt.Errorf("applyUpdates: failed to update matured balances: %w", err) - } - sources := make(map[types.SiacoinOutputID]Source) for i := range cau.Block.MinerPayouts { sources[cau.Block.ID().MinerOutputID(i)] = SourceMinerPayout @@ -142,49 +144,27 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { }) }) - scDBIds, err := tx.AddSiacoinElements( - cau.Block.ID(), - sources, - append(spentSiacoinElements, ephemeralSiacoinElements...), - newSiacoinElements, - ) - if err != nil { - return fmt.Errorf("applyUpdates: failed to add siacoin outputs: %w", err) - } - sfDBIds, err := tx.AddSiafundElements( - cau.Block.ID(), - append(spentSiafundElements, ephemeralSiafundElements...), - newSiafundElements, - ) - if err != nil { - return fmt.Errorf("applyUpdates: failed to add siafund outputs: %w", err) - } - if err := tx.UpdateBalances(cau.State.Index.Height, spentSiacoinElements, newSiacoinElements, spentSiafundElements, newSiafundElements); err != nil { - return fmt.Errorf("applyUpdates: failed to update balances: %w", err) - } + state := UpdateState{ + Block: cau.Block, + Index: cau.State.Index, + TreeUpdates: treeUpdates, - fcDBIds, err := tx.AddFileContractElements(cau.Block.ID(), fces) - if err != nil { - return fmt.Errorf("applyUpdates: failed to add file contracts: %w", err) - } + Sources: sources, + NewSiacoinElements: newSiacoinElements, + SpentSiacoinElements: spentSiacoinElements, + EphemeralSiacoinElements: ephemeralSiacoinElements, - if err := tx.AddMinerPayouts(cau.Block.ID(), cau.State.Index.Height, cau.Block.MinerPayouts, scDBIds); err != nil { - return fmt.Errorf("applyUpdates: failed to add miner payouts: %w", err) - } else if err := tx.AddTransactions(cau.Block.ID(), cau.Block.Transactions, scDBIds, sfDBIds, fcDBIds); err != nil { - return fmt.Errorf("applyUpdates: failed to add transactions: addTransactions: %w", err) - } else if err := tx.UpdateStateTree(treeUpdates); err != nil { - return fmt.Errorf("applyUpdates: failed to update state tree: %w", err) - } + NewSiafundElements: newSiafundElements, + SpentSiafundElements: spentSiafundElements, + EphemeralSiafundElements: ephemeralSiafundElements, - return nil + FileContractElements: fces, + } + return tx.ApplyIndex(state) } // revertChainUpdate atomically reverts a chain update from a store func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types.ChainIndex) error { - if err := tx.UpdateMaturedBalances(true, revertedIndex.Height); err != nil { - return fmt.Errorf("revertUpdate: failed to update matured balances: %w", err) - } - created := make(map[types.Hash256]bool) ephemeral := make(map[types.Hash256]bool) for _, txn := range cru.Block.Transactions { @@ -252,30 +232,22 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. }) }) - // log.Println("REVERT!") - if _, err := tx.AddSiacoinElements( - cru.Block.ID(), - nil, - spentSiacoinElements, - append(newSiacoinElements, ephemeralSiacoinElements...), - ); err != nil { - return fmt.Errorf("revertUpdate: failed to update siacoin output state: %w", err) - } else if _, err := tx.AddSiafundElements( - cru.Block.ID(), - spentSiafundElements, - append(newSiafundElements, ephemeralSiafundElements...), - ); err != nil { - return fmt.Errorf("revertUpdate: failed to update siafund output state: %w", err) - } else if err := tx.UpdateBalances(revertedIndex.Height, spentSiacoinElements, newSiacoinElements, spentSiafundElements, newSiafundElements); err != nil { - return fmt.Errorf("revertUpdate: failed to update balances: %w", err) - } else if _, err := tx.AddFileContractElements(cru.Block.ID(), fces); err != nil { - return fmt.Errorf("revertUpdate: failed to update file contract state: %w", err) - } else if err := tx.DeleteBlock(cru.Block.ID()); err != nil { - return fmt.Errorf("revertUpdate: failed to delete block: %w", err) - } else if err := tx.UpdateStateTree(treeUpdates); err != nil { - return fmt.Errorf("revertUpdate: failed to update state tree: %w", err) + state := UpdateState{ + Block: cru.Block, + Index: revertedIndex, + TreeUpdates: treeUpdates, + + NewSiacoinElements: newSiacoinElements, + SpentSiacoinElements: spentSiacoinElements, + EphemeralSiacoinElements: ephemeralSiacoinElements, + + NewSiafundElements: newSiafundElements, + SpentSiafundElements: spentSiafundElements, + EphemeralSiafundElements: ephemeralSiafundElements, + + FileContractElements: fces, } - return nil + return tx.RevertIndex(state) } // UpdateChainState applies the reverts and updates. diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index f9530c9d..cd419e74 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -530,6 +530,11 @@ func (ut *updateTx) AddSiafundElements(bid types.BlockID, spentElements, newElem return sfDBIds, nil } +func (ut *updateTx) DeleteBlock(bid types.BlockID) error { + _, err := ut.tx.Exec("DELETE FROM blocks WHERE id = ?", encode(bid)) + return err +} + func (ut *updateTx) AddFileContractElements(bid types.BlockID, fces []explorer.FileContractUpdate) (map[explorer.DBFileContract]int64, error) { stmt, err := ut.tx.Prepare(`INSERT INTO file_contract_elements(block_id, contract_id, leaf_index, resolved, valid, filesize, file_merkle_root, window_start, window_end, payout, unlock_hash, revision_number) VALUES (?, ?, ?, FALSE, TRUE, ?, ?, ?, ?, ?, ?, ?) @@ -574,9 +579,77 @@ func (ut *updateTx) AddFileContractElements(bid types.BlockID, fces []explorer.F return fcDBIds, updateErr } -func (ut *updateTx) DeleteBlock(bid types.BlockID) error { - _, err := ut.tx.Exec("DELETE FROM blocks WHERE id = ?", encode(bid)) - return err +func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { + if err := ut.AddBlock(state.Block, state.Index.Height); err != nil { + return fmt.Errorf("ApplyIndex: failed to add block: %w", err) + } else if err := ut.UpdateMaturedBalances(false, state.Index.Height); err != nil { + return fmt.Errorf("ApplyIndex: failed to update matured balances: %w", err) + } + + scDBIds, err := ut.AddSiacoinElements( + state.Block.ID(), + state.Sources, + append(state.SpentSiacoinElements, state.EphemeralSiacoinElements...), + state.NewSiacoinElements, + ) + if err != nil { + return fmt.Errorf("ApplyIndex: failed to add siacoin outputs: %w", err) + } + sfDBIds, err := ut.AddSiafundElements( + state.Block.ID(), + append(state.SpentSiafundElements, state.EphemeralSiafundElements...), + state.NewSiafundElements, + ) + if err != nil { + return fmt.Errorf("ApplyIndex: failed to add siafund outputs: %w", err) + } + if err := ut.UpdateBalances(state.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { + return fmt.Errorf("ApplyIndex: failed to update balances: %w", err) + } + + fcDBIds, err := ut.AddFileContractElements(state.Block.ID(), state.FileContractElements) + if err != nil { + return fmt.Errorf("v: failed to add file contracts: %w", err) + } + + if err := ut.AddMinerPayouts(state.Block.ID(), state.Index.Height, state.Block.MinerPayouts, scDBIds); err != nil { + return fmt.Errorf("ApplyIndex: failed to add miner payouts: %w", err) + } else if err := ut.AddTransactions(state.Block.ID(), state.Block.Transactions, scDBIds, sfDBIds, fcDBIds); err != nil { + return fmt.Errorf("ApplyIndex: failed to add transactions: addTransactions: %w", err) + } else if err := ut.UpdateStateTree(state.TreeUpdates); err != nil { + return fmt.Errorf("ApplyIndex: failed to update state tree: %w", err) + } + + return nil +} + +func (ut *updateTx) RevertIndex(state explorer.UpdateState) error { + if err := ut.UpdateMaturedBalances(true, state.Index.Height); err != nil { + return fmt.Errorf("RevertIndex: failed to update matured balances: %w", err) + } else if _, err := ut.AddSiacoinElements( + state.Block.ID(), + nil, + state.SpentSiacoinElements, + append(state.NewSiacoinElements, state.EphemeralSiacoinElements...), + ); err != nil { + return fmt.Errorf("RevertIndex: failed to update siacoin output state: %w", err) + } else if _, err := ut.AddSiafundElements( + state.Block.ID(), + state.SpentSiafundElements, + append(state.NewSiafundElements, state.EphemeralSiafundElements...), + ); err != nil { + return fmt.Errorf("RevertIndex: failed to update siafund output state: %w", err) + } else if err := ut.UpdateBalances(state.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { + return fmt.Errorf("RevertIndex: failed to update balances: %w", err) + } else if _, err := ut.AddFileContractElements(state.Block.ID(), state.FileContractElements); err != nil { + return fmt.Errorf("RevertIndex: failed to update file contract state: %w", err) + } else if err := ut.DeleteBlock(state.Block.ID()); err != nil { + return fmt.Errorf("RevertIndex: failed to delete block: %w", err) + } else if err := ut.UpdateStateTree(state.TreeUpdates); err != nil { + return fmt.Errorf("RevertIndex: failed to update state tree: %w", err) + } + + return nil } // UpdateChainState implements explorer.Store From a77d6f988e11e42e738688db8513d97686050713 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Thu, 25 Apr 2024 23:14:56 -0400 Subject: [PATCH 13/19] only export what's necessary on updateTx --- persist/sqlite/consensus.go | 52 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index cd419e74..61c3f38d 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -15,13 +15,13 @@ type updateTx struct { relevantAddresses map[types.Address]bool } -func (ut *updateTx) AddBlock(b types.Block, height uint64) error { +func (ut *updateTx) addBlock(b types.Block, height uint64) error { // nonce is encoded because database/sql doesn't support uint64 with high bit set _, err := ut.tx.Exec("INSERT INTO blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encode(b.Nonce), encode(b.Timestamp)) return err } -func (ut *updateTx) AddMinerPayouts(bid types.BlockID, height uint64, scos []types.SiacoinOutput, dbIDs map[types.SiacoinOutputID]int64) error { +func (ut *updateTx) addMinerPayouts(bid types.BlockID, height uint64, scos []types.SiacoinOutput, dbIDs map[types.SiacoinOutputID]int64) error { stmt, err := ut.tx.Prepare(`INSERT INTO miner_payouts(block_id, block_order, output_id) VALUES (?, ?, ?);`) if err != nil { return fmt.Errorf("addMinerPayouts: failed to prepare statement: %w", err) @@ -217,7 +217,7 @@ func (ut *updateTx) addFileContractRevisions(id int64, txn types.Transaction, db return nil } -func (ut *updateTx) AddTransactions(bid types.BlockID, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[explorer.DBFileContract]int64) error { +func (ut *updateTx) addTransactions(bid types.BlockID, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[explorer.DBFileContract]int64) error { insertTransactionStmt, err := ut.tx.Prepare(`INSERT INTO transactions (transaction_id) VALUES (?) ON CONFLICT (transaction_id) DO UPDATE SET transaction_id=EXCLUDED.transaction_id -- technically a no-op, but necessary for the RETURNING clause RETURNING id;`) @@ -266,7 +266,7 @@ type balance struct { sf uint64 } -func (ut *updateTx) UpdateBalances(height uint64, spentSiacoinElements, newSiacoinElements []types.SiacoinElement, spentSiafundElements, newSiafundElements []types.SiafundElement) error { +func (ut *updateTx) updateBalances(height uint64, spentSiacoinElements, newSiacoinElements []types.SiacoinElement, spentSiafundElements, newSiafundElements []types.SiafundElement) error { addresses := make(map[types.Address]balance) for _, sce := range spentSiacoinElements { addresses[sce.SiacoinOutput.Address] = balance{} @@ -355,7 +355,7 @@ func (ut *updateTx) UpdateBalances(height uint64, spentSiacoinElements, newSiaco return nil } -func (ut *updateTx) UpdateMaturedBalances(revert bool, height uint64) error { +func (ut *updateTx) updateMaturedBalances(revert bool, height uint64) error { // Prevent double counting - outputs with a maturity height of 0 are // handled in updateBalances if height == 0 { @@ -432,7 +432,7 @@ func (ut *updateTx) UpdateMaturedBalances(revert bool, height uint64) error { return nil } -func (ut *updateTx) UpdateStateTree(changes []explorer.TreeNodeUpdate) error { +func (ut *updateTx) updateStateTree(changes []explorer.TreeNodeUpdate) error { stmt, err := ut.tx.Prepare(`INSERT INTO state_tree (row, column, value) VALUES($1, $2, $3) ON CONFLICT (row, column) DO UPDATE SET value=EXCLUDED.value;`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) @@ -448,7 +448,7 @@ func (ut *updateTx) UpdateStateTree(changes []explorer.TreeNodeUpdate) error { return nil } -func (ut *updateTx) AddSiacoinElements(bid types.BlockID, sources map[types.SiacoinOutputID]explorer.Source, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) { +func (ut *updateTx) addSiacoinElements(bid types.BlockID, sources map[types.SiacoinOutputID]explorer.Source, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) { stmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements(output_id, block_id, leaf_index, spent, source, maturity_height, address, value) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (output_id) @@ -489,7 +489,7 @@ func (ut *updateTx) AddSiacoinElements(bid types.BlockID, sources map[types.Siac return scDBIds, nil } -func (ut *updateTx) AddSiafundElements(bid types.BlockID, spentElements, newElements []types.SiafundElement) (map[types.SiafundOutputID]int64, error) { +func (ut *updateTx) addSiafundElements(bid types.BlockID, spentElements, newElements []types.SiafundElement) (map[types.SiafundOutputID]int64, error) { stmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements(output_id, block_id, leaf_index, spent, claim_start, address, value) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT @@ -530,12 +530,12 @@ func (ut *updateTx) AddSiafundElements(bid types.BlockID, spentElements, newElem return sfDBIds, nil } -func (ut *updateTx) DeleteBlock(bid types.BlockID) error { +func (ut *updateTx) deleteBlock(bid types.BlockID) error { _, err := ut.tx.Exec("DELETE FROM blocks WHERE id = ?", encode(bid)) return err } -func (ut *updateTx) AddFileContractElements(bid types.BlockID, fces []explorer.FileContractUpdate) (map[explorer.DBFileContract]int64, error) { +func (ut *updateTx) addFileContractElements(bid types.BlockID, fces []explorer.FileContractUpdate) (map[explorer.DBFileContract]int64, error) { stmt, err := ut.tx.Prepare(`INSERT INTO file_contract_elements(block_id, contract_id, leaf_index, resolved, valid, filesize, file_merkle_root, window_start, window_end, payout, unlock_hash, revision_number) VALUES (?, ?, ?, FALSE, TRUE, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (contract_id, revision_number) @@ -580,13 +580,13 @@ func (ut *updateTx) AddFileContractElements(bid types.BlockID, fces []explorer.F } func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { - if err := ut.AddBlock(state.Block, state.Index.Height); err != nil { + if err := ut.addBlock(state.Block, state.Index.Height); err != nil { return fmt.Errorf("ApplyIndex: failed to add block: %w", err) - } else if err := ut.UpdateMaturedBalances(false, state.Index.Height); err != nil { + } else if err := ut.updateMaturedBalances(false, state.Index.Height); err != nil { return fmt.Errorf("ApplyIndex: failed to update matured balances: %w", err) } - scDBIds, err := ut.AddSiacoinElements( + scDBIds, err := ut.addSiacoinElements( state.Block.ID(), state.Sources, append(state.SpentSiacoinElements, state.EphemeralSiacoinElements...), @@ -595,7 +595,7 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { if err != nil { return fmt.Errorf("ApplyIndex: failed to add siacoin outputs: %w", err) } - sfDBIds, err := ut.AddSiafundElements( + sfDBIds, err := ut.addSiafundElements( state.Block.ID(), append(state.SpentSiafundElements, state.EphemeralSiafundElements...), state.NewSiafundElements, @@ -603,20 +603,20 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { if err != nil { return fmt.Errorf("ApplyIndex: failed to add siafund outputs: %w", err) } - if err := ut.UpdateBalances(state.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { + if err := ut.updateBalances(state.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { return fmt.Errorf("ApplyIndex: failed to update balances: %w", err) } - fcDBIds, err := ut.AddFileContractElements(state.Block.ID(), state.FileContractElements) + fcDBIds, err := ut.addFileContractElements(state.Block.ID(), state.FileContractElements) if err != nil { return fmt.Errorf("v: failed to add file contracts: %w", err) } - if err := ut.AddMinerPayouts(state.Block.ID(), state.Index.Height, state.Block.MinerPayouts, scDBIds); err != nil { + if err := ut.addMinerPayouts(state.Block.ID(), state.Index.Height, state.Block.MinerPayouts, scDBIds); err != nil { return fmt.Errorf("ApplyIndex: failed to add miner payouts: %w", err) - } else if err := ut.AddTransactions(state.Block.ID(), state.Block.Transactions, scDBIds, sfDBIds, fcDBIds); err != nil { + } else if err := ut.addTransactions(state.Block.ID(), state.Block.Transactions, scDBIds, sfDBIds, fcDBIds); err != nil { return fmt.Errorf("ApplyIndex: failed to add transactions: addTransactions: %w", err) - } else if err := ut.UpdateStateTree(state.TreeUpdates); err != nil { + } else if err := ut.updateStateTree(state.TreeUpdates); err != nil { return fmt.Errorf("ApplyIndex: failed to update state tree: %w", err) } @@ -624,28 +624,28 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { } func (ut *updateTx) RevertIndex(state explorer.UpdateState) error { - if err := ut.UpdateMaturedBalances(true, state.Index.Height); err != nil { + if err := ut.updateMaturedBalances(true, state.Index.Height); err != nil { return fmt.Errorf("RevertIndex: failed to update matured balances: %w", err) - } else if _, err := ut.AddSiacoinElements( + } else if _, err := ut.addSiacoinElements( state.Block.ID(), nil, state.SpentSiacoinElements, append(state.NewSiacoinElements, state.EphemeralSiacoinElements...), ); err != nil { return fmt.Errorf("RevertIndex: failed to update siacoin output state: %w", err) - } else if _, err := ut.AddSiafundElements( + } else if _, err := ut.addSiafundElements( state.Block.ID(), state.SpentSiafundElements, append(state.NewSiafundElements, state.EphemeralSiafundElements...), ); err != nil { return fmt.Errorf("RevertIndex: failed to update siafund output state: %w", err) - } else if err := ut.UpdateBalances(state.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { + } else if err := ut.updateBalances(state.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { return fmt.Errorf("RevertIndex: failed to update balances: %w", err) - } else if _, err := ut.AddFileContractElements(state.Block.ID(), state.FileContractElements); err != nil { + } else if _, err := ut.addFileContractElements(state.Block.ID(), state.FileContractElements); err != nil { return fmt.Errorf("RevertIndex: failed to update file contract state: %w", err) - } else if err := ut.DeleteBlock(state.Block.ID()); err != nil { + } else if err := ut.deleteBlock(state.Block.ID()); err != nil { return fmt.Errorf("RevertIndex: failed to delete block: %w", err) - } else if err := ut.UpdateStateTree(state.TreeUpdates); err != nil { + } else if err := ut.updateStateTree(state.TreeUpdates); err != nil { return fmt.Errorf("RevertIndex: failed to update state tree: %w", err) } From cf76455f8988472c6145f06ef241700cc48ba8fc Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Fri, 26 Apr 2024 14:33:48 -0400 Subject: [PATCH 14/19] use Source field of SiacoinOutput instead of sources map --- explorer/update.go | 43 ++++++++++++++++++++++++------------- persist/sqlite/consensus.go | 10 ++++----- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/explorer/update.go b/explorer/update.go index 948b2fa6..4c23a813 100644 --- a/explorer/update.go +++ b/explorer/update.go @@ -35,10 +35,9 @@ type ( Index types.ChainIndex TreeUpdates []TreeNodeUpdate - Sources map[types.SiacoinOutputID]Source - NewSiacoinElements []types.SiacoinElement - SpentSiacoinElements []types.SiacoinElement - EphemeralSiacoinElements []types.SiacoinElement + NewSiacoinElements []SiacoinOutput + SpentSiacoinElements []SiacoinOutput + EphemeralSiacoinElements []SiacoinOutput NewSiafundElements []types.SiafundElement SpentSiafundElements []types.SiafundElement @@ -95,18 +94,27 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { } // add new siacoin elements to the store - var newSiacoinElements, spentSiacoinElements []types.SiacoinElement - var ephemeralSiacoinElements []types.SiacoinElement + var newSiacoinElements, spentSiacoinElements []SiacoinOutput + var ephemeralSiacoinElements []SiacoinOutput cau.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { if ephemeral[se.ID] { - ephemeralSiacoinElements = append(ephemeralSiacoinElements, se) + ephemeralSiacoinElements = append(ephemeralSiacoinElements, SiacoinOutput{ + SiacoinElement: se, + Source: sources[types.SiacoinOutputID(se.StateElement.ID)], + }) return } if spent { - spentSiacoinElements = append(spentSiacoinElements, se) + spentSiacoinElements = append(spentSiacoinElements, SiacoinOutput{ + SiacoinElement: se, + Source: sources[types.SiacoinOutputID(se.StateElement.ID)], + }) } else { - newSiacoinElements = append(newSiacoinElements, se) + newSiacoinElements = append(newSiacoinElements, SiacoinOutput{ + SiacoinElement: se, + Source: sources[types.SiacoinOutputID(se.StateElement.ID)], + }) } }) @@ -149,7 +157,6 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { Index: cau.State.Index, TreeUpdates: treeUpdates, - Sources: sources, NewSiacoinElements: newSiacoinElements, SpentSiacoinElements: spentSiacoinElements, EphemeralSiacoinElements: ephemeralSiacoinElements, @@ -183,18 +190,24 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. } // add new siacoin elements to the store - var newSiacoinElements, spentSiacoinElements []types.SiacoinElement - var ephemeralSiacoinElements []types.SiacoinElement + var newSiacoinElements, spentSiacoinElements []SiacoinOutput + var ephemeralSiacoinElements []SiacoinOutput cru.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { if ephemeral[se.ID] { - ephemeralSiacoinElements = append(ephemeralSiacoinElements, se) + ephemeralSiacoinElements = append(ephemeralSiacoinElements, SiacoinOutput{ + SiacoinElement: se, + }) return } if spent { - newSiacoinElements = append(newSiacoinElements, se) + newSiacoinElements = append(newSiacoinElements, SiacoinOutput{ + SiacoinElement: se, + }) } else { - spentSiacoinElements = append(spentSiacoinElements, se) + spentSiacoinElements = append(spentSiacoinElements, SiacoinOutput{ + SiacoinElement: se, + }) } }) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 61c3f38d..dbf34c3c 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -266,7 +266,7 @@ type balance struct { sf uint64 } -func (ut *updateTx) updateBalances(height uint64, spentSiacoinElements, newSiacoinElements []types.SiacoinElement, spentSiafundElements, newSiafundElements []types.SiafundElement) error { +func (ut *updateTx) updateBalances(height uint64, spentSiacoinElements, newSiacoinElements []explorer.SiacoinOutput, spentSiafundElements, newSiafundElements []types.SiafundElement) error { addresses := make(map[types.Address]balance) for _, sce := range spentSiacoinElements { addresses[sce.SiacoinOutput.Address] = balance{} @@ -448,7 +448,7 @@ func (ut *updateTx) updateStateTree(changes []explorer.TreeNodeUpdate) error { return nil } -func (ut *updateTx) addSiacoinElements(bid types.BlockID, sources map[types.SiacoinOutputID]explorer.Source, spentElements, newElements []types.SiacoinElement) (map[types.SiacoinOutputID]int64, error) { +func (ut *updateTx) addSiacoinElements(bid types.BlockID, spentElements, newElements []explorer.SiacoinOutput) (map[types.SiacoinOutputID]int64, error) { stmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements(output_id, block_id, leaf_index, spent, source, maturity_height, address, value) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (output_id) @@ -460,7 +460,7 @@ func (ut *updateTx) addSiacoinElements(bid types.BlockID, sources map[types.Siac scDBIds := make(map[types.SiacoinOutputID]int64) for _, sce := range newElements { - result, err := stmt.Exec(encode(sce.StateElement.ID), encode(bid), encode(sce.StateElement.LeafIndex), false, int(sources[types.SiacoinOutputID(sce.StateElement.ID)]), sce.MaturityHeight, encode(sce.SiacoinOutput.Address), encode(sce.SiacoinOutput.Value), false, encode(sce.StateElement.LeafIndex)) + result, err := stmt.Exec(encode(sce.StateElement.ID), encode(bid), encode(sce.StateElement.LeafIndex), false, int(sce.Source), sce.MaturityHeight, encode(sce.SiacoinOutput.Address), encode(sce.SiacoinOutput.Value), false, encode(sce.StateElement.LeafIndex)) if err != nil { return nil, fmt.Errorf("addSiacoinElements: failed to execute siacoin_elements statement: %w", err) } @@ -473,7 +473,7 @@ func (ut *updateTx) addSiacoinElements(bid types.BlockID, sources map[types.Siac scDBIds[types.SiacoinOutputID(sce.StateElement.ID)] = dbID } for _, sce := range spentElements { - result, err := stmt.Exec(encode(sce.StateElement.ID), encode(bid), encode(sce.StateElement.LeafIndex), true, int(sources[types.SiacoinOutputID(sce.StateElement.ID)]), sce.MaturityHeight, encode(sce.SiacoinOutput.Address), encode(sce.SiacoinOutput.Value), true, encode(sce.StateElement.LeafIndex)) + result, err := stmt.Exec(encode(sce.StateElement.ID), encode(bid), encode(sce.StateElement.LeafIndex), true, int(sce.Source), sce.MaturityHeight, encode(sce.SiacoinOutput.Address), encode(sce.SiacoinOutput.Value), true, encode(sce.StateElement.LeafIndex)) if err != nil { return nil, fmt.Errorf("addSiacoinElements: failed to execute siacoin_elements statement: %w", err) } @@ -588,7 +588,6 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { scDBIds, err := ut.addSiacoinElements( state.Block.ID(), - state.Sources, append(state.SpentSiacoinElements, state.EphemeralSiacoinElements...), state.NewSiacoinElements, ) @@ -628,7 +627,6 @@ func (ut *updateTx) RevertIndex(state explorer.UpdateState) error { return fmt.Errorf("RevertIndex: failed to update matured balances: %w", err) } else if _, err := ut.addSiacoinElements( state.Block.ID(), - nil, state.SpentSiacoinElements, append(state.NewSiacoinElements, state.EphemeralSiacoinElements...), ); err != nil { From 3925ac4400edb7a4162d79dcc0d2fd1778c34acd Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Mon, 29 Apr 2024 15:49:39 -0400 Subject: [PATCH 15/19] fix bug where transactions already stored in the DB would unnecessarily have their fields reinserted --- persist/sqlite/consensus.go | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index dbf34c3c..deea915e 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -218,9 +218,12 @@ func (ut *updateTx) addFileContractRevisions(id int64, txn types.Transaction, db } func (ut *updateTx) addTransactions(bid types.BlockID, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[explorer.DBFileContract]int64) error { - insertTransactionStmt, err := ut.tx.Prepare(`INSERT INTO transactions (transaction_id) VALUES (?) - ON CONFLICT (transaction_id) DO UPDATE SET transaction_id=EXCLUDED.transaction_id -- technically a no-op, but necessary for the RETURNING clause - RETURNING id;`) + checkTransactionStmt, err := ut.tx.Prepare(`SELECT id FROM transactions WHERE transaction_id = ?`) + if err != nil { + return fmt.Errorf("failed to prepare check transaction statement: %v", err) + } + + insertTransactionStmt, err := ut.tx.Prepare(`INSERT INTO transactions (transaction_id) VALUES (?)`) if err != nil { return fmt.Errorf("failed to prepare insert transaction statement: %v", err) } @@ -233,15 +236,34 @@ func (ut *updateTx) addTransactions(bid types.BlockID, txns []types.Transaction, defer blockTransactionsStmt.Close() for i, txn := range txns { + var exist bool var txnID int64 - err := insertTransactionStmt.QueryRow(encode(txn.ID())).Scan(&txnID) - if err != nil { - return fmt.Errorf("failed to insert into transactions: %w", err) + if err := checkTransactionStmt.QueryRow(encode(txn.ID())).Scan(&txnID); err != nil && err != sql.ErrNoRows { + return fmt.Errorf("failed to insert transaction ID: %w", err) + } else if err == nil { + exist = true + } + + if !exist { + result, err := insertTransactionStmt.Exec(encode(txn.ID())) + if err != nil { + return fmt.Errorf("failed to insert into transactions: %w", err) + } + txnID, err = result.LastInsertId() + if err != nil { + return fmt.Errorf("failed to get transaction ID: %w", err) + } } if _, err := blockTransactionsStmt.Exec(encode(bid), txnID, i); err != nil { return fmt.Errorf("failed to insert into block_transactions: %w", err) - } else if err := ut.addArbitraryData(txnID, txn); err != nil { + } + + // transaction already exists, don't reinsert its fields + if exist { + continue + } + if err := ut.addArbitraryData(txnID, txn); err != nil { return fmt.Errorf("failed to add arbitrary data: %w", err) } else if err := ut.addSiacoinInputs(txnID, txn); err != nil { return fmt.Errorf("failed to add siacoin inputs: %w", err) From 36240318703da849a4984e66cd372330f2cae5d9 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Tue, 30 Apr 2024 11:10:55 -0400 Subject: [PATCH 16/19] remove unused --- persist/sqlite/consensus.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index deea915e..7085628d 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -11,8 +11,7 @@ import ( ) type updateTx struct { - tx *txn - relevantAddresses map[types.Address]bool + tx *txn } func (ut *updateTx) addBlock(b types.Block, height uint64) error { @@ -21,7 +20,7 @@ func (ut *updateTx) addBlock(b types.Block, height uint64) error { return err } -func (ut *updateTx) addMinerPayouts(bid types.BlockID, height uint64, scos []types.SiacoinOutput, dbIDs map[types.SiacoinOutputID]int64) error { +func (ut *updateTx) addMinerPayouts(bid types.BlockID, scos []types.SiacoinOutput, dbIDs map[types.SiacoinOutputID]int64) error { stmt, err := ut.tx.Prepare(`INSERT INTO miner_payouts(block_id, block_order, output_id) VALUES (?, ?, ?);`) if err != nil { return fmt.Errorf("addMinerPayouts: failed to prepare statement: %w", err) @@ -633,7 +632,7 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { return fmt.Errorf("v: failed to add file contracts: %w", err) } - if err := ut.addMinerPayouts(state.Block.ID(), state.Index.Height, state.Block.MinerPayouts, scDBIds); err != nil { + if err := ut.addMinerPayouts(state.Block.ID(), state.Block.MinerPayouts, scDBIds); err != nil { return fmt.Errorf("ApplyIndex: failed to add miner payouts: %w", err) } else if err := ut.addTransactions(state.Block.ID(), state.Block.Transactions, scDBIds, sfDBIds, fcDBIds); err != nil { return fmt.Errorf("ApplyIndex: failed to add transactions: addTransactions: %w", err) @@ -676,8 +675,7 @@ func (ut *updateTx) RevertIndex(state explorer.UpdateState) error { func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error { return s.transaction(func(tx *txn) error { utx := &updateTx{ - tx: tx, - relevantAddresses: make(map[types.Address]bool), + tx: tx, } if err := explorer.UpdateChainState(utx, reverted, applied); err != nil { From 47e561c69dabdab69204d40f87dd13fd0fb62482 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Tue, 30 Apr 2024 11:11:27 -0400 Subject: [PATCH 17/19] use big endian for uint64 encoding --- persist/sqlite/encoding.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/persist/sqlite/encoding.go b/persist/sqlite/encoding.go index 966c6b00..8788255d 100644 --- a/persist/sqlite/encoding.go +++ b/persist/sqlite/encoding.go @@ -27,7 +27,7 @@ func encode(obj any) any { return buf.Bytes() case uint64: b := make([]byte, 8) - binary.LittleEndian.PutUint64(b, obj) + binary.BigEndian.PutUint64(b, obj) return b case time.Time: return obj.Unix() @@ -60,7 +60,7 @@ func (d *decodable) Scan(src any) error { v.DecodeFrom(dec) return dec.Err() case *uint64: - *v = binary.LittleEndian.Uint64(src) + *v = binary.BigEndian.Uint64(src) default: return fmt.Errorf("cannot scan %T to %T", src, d.v) } From 060ae743cc5fa0869619f69438b13e7abaa157bb Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Tue, 30 Apr 2024 11:12:18 -0400 Subject: [PATCH 18/19] remove slice encoding/decoding --- persist/sqlite/encoding.go | 43 -------------------------------------- 1 file changed, 43 deletions(-) diff --git a/persist/sqlite/encoding.go b/persist/sqlite/encoding.go index 8788255d..7fe4d0e0 100644 --- a/persist/sqlite/encoding.go +++ b/persist/sqlite/encoding.go @@ -83,46 +83,3 @@ func (d *decodable) Scan(src any) error { func decode(obj any) sql.Scanner { return &decodable{obj} } - -type decodableSlice[T any] struct { - v *[]T -} - -func (d *decodableSlice[T]) Scan(src any) error { - switch src := src.(type) { - case []byte: - dec := types.NewBufDecoder(src) - s := make([]T, dec.ReadPrefix()) - for i := range s { - dv, ok := any(&s[i]).(types.DecoderFrom) - if !ok { - panic(fmt.Errorf("cannot decode %T", s[i])) - } - dv.DecodeFrom(dec) - } - if err := dec.Err(); err != nil { - return err - } - *d.v = s - return nil - default: - return fmt.Errorf("cannot scan %T to []byte", src) - } -} - -func decodeSlice[T any](v *[]T) sql.Scanner { - return &decodableSlice[T]{v: v} -} - -func encodeSlice[T types.EncoderTo](v []T) []byte { - var buf bytes.Buffer - enc := types.NewEncoder(&buf) - enc.WritePrefix(len(v)) - for _, e := range v { - e.EncodeTo(enc) - } - if err := enc.Flush(); err != nil { - panic(err) - } - return buf.Bytes() -} From f247b608b3199f295ba52258131ac3f8b2828937 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Tue, 30 Apr 2024 11:19:47 -0400 Subject: [PATCH 19/19] move lower level functions off updateTx --- persist/sqlite/consensus.go | 138 +++++++++++++++++++----------------- 1 file changed, 71 insertions(+), 67 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 7085628d..b1330b55 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -14,14 +14,14 @@ type updateTx struct { tx *txn } -func (ut *updateTx) addBlock(b types.Block, height uint64) error { +func addBlock(tx *txn, b types.Block, height uint64) error { // nonce is encoded because database/sql doesn't support uint64 with high bit set - _, err := ut.tx.Exec("INSERT INTO blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encode(b.Nonce), encode(b.Timestamp)) + _, err := tx.Exec("INSERT INTO blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encode(b.Nonce), encode(b.Timestamp)) return err } -func (ut *updateTx) addMinerPayouts(bid types.BlockID, scos []types.SiacoinOutput, dbIDs map[types.SiacoinOutputID]int64) error { - stmt, err := ut.tx.Prepare(`INSERT INTO miner_payouts(block_id, block_order, output_id) VALUES (?, ?, ?);`) +func addMinerPayouts(tx *txn, bid types.BlockID, scos []types.SiacoinOutput, dbIDs map[types.SiacoinOutputID]int64) error { + stmt, err := tx.Prepare(`INSERT INTO miner_payouts(block_id, block_order, output_id) VALUES (?, ?, ?);`) if err != nil { return fmt.Errorf("addMinerPayouts: failed to prepare statement: %w", err) } @@ -40,8 +40,8 @@ func (ut *updateTx) addMinerPayouts(bid types.BlockID, scos []types.SiacoinOutpu return nil } -func (ut *updateTx) addArbitraryData(id int64, txn types.Transaction) error { - stmt, err := ut.tx.Prepare(`INSERT INTO transaction_arbitrary_data(transaction_id, transaction_order, data) VALUES (?, ?, ?)`) +func addArbitraryData(tx *txn, id int64, txn types.Transaction) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_arbitrary_data(transaction_id, transaction_order, data) VALUES (?, ?, ?)`) if err != nil { return fmt.Errorf("addArbitraryData: failed to prepare statement: %w", err) @@ -56,8 +56,8 @@ func (ut *updateTx) addArbitraryData(id int64, txn types.Transaction) error { return nil } -func (ut *updateTx) addSiacoinInputs(id int64, txn types.Transaction) error { - stmt, err := ut.tx.Prepare(`INSERT INTO transaction_siacoin_inputs(transaction_id, transaction_order, parent_id, unlock_conditions) VALUES (?, ?, ?, ?)`) +func addSiacoinInputs(tx *txn, id int64, txn types.Transaction) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_siacoin_inputs(transaction_id, transaction_order, parent_id, unlock_conditions) VALUES (?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addSiacoinInputs: failed to prepare statement: %w", err) } @@ -71,8 +71,8 @@ func (ut *updateTx) addSiacoinInputs(id int64, txn types.Transaction) error { return nil } -func (ut *updateTx) addSiacoinOutputs(id int64, txn types.Transaction, dbIDs map[types.SiacoinOutputID]int64) error { - stmt, err := ut.tx.Prepare(`INSERT INTO transaction_siacoin_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, ?)`) +func addSiacoinOutputs(tx *txn, id int64, txn types.Transaction, dbIDs map[types.SiacoinOutputID]int64) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_siacoin_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, ?)`) if err != nil { return fmt.Errorf("addSiacoinOutputs: failed to prepare statement: %w", err) } @@ -91,8 +91,8 @@ func (ut *updateTx) addSiacoinOutputs(id int64, txn types.Transaction, dbIDs map return nil } -func (ut *updateTx) addSiafundInputs(id int64, txn types.Transaction) error { - stmt, err := ut.tx.Prepare(`INSERT INTO transaction_siafund_inputs(transaction_id, transaction_order, parent_id, unlock_conditions, claim_address) VALUES (?, ?, ?, ?, ?)`) +func addSiafundInputs(tx *txn, id int64, txn types.Transaction) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_siafund_inputs(transaction_id, transaction_order, parent_id, unlock_conditions, claim_address) VALUES (?, ?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addSiafundInputs: failed to prepare statement: %w", err) } @@ -106,8 +106,8 @@ func (ut *updateTx) addSiafundInputs(id int64, txn types.Transaction) error { return nil } -func (ut *updateTx) addSiafundOutputs(id int64, txn types.Transaction, dbIDs map[types.SiafundOutputID]int64) error { - stmt, err := ut.tx.Prepare(`INSERT INTO transaction_siafund_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, ?)`) +func addSiafundOutputs(tx *txn, id int64, txn types.Transaction, dbIDs map[types.SiafundOutputID]int64) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_siafund_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, ?)`) if err != nil { return fmt.Errorf("addSiafundOutputs: failed to prepare statement: %w", err) } @@ -126,20 +126,20 @@ func (ut *updateTx) addSiafundOutputs(id int64, txn types.Transaction, dbIDs map return nil } -func (ut *updateTx) addFileContracts(id int64, txn types.Transaction, fcDBIds map[explorer.DBFileContract]int64) error { - stmt, err := ut.tx.Prepare(`INSERT INTO transaction_file_contracts(transaction_id, transaction_order, contract_id) VALUES (?, ?, ?)`) +func addFileContracts(tx *txn, id int64, txn types.Transaction, fcDBIds map[explorer.DBFileContract]int64) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_file_contracts(transaction_id, transaction_order, contract_id) VALUES (?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContracts: failed to prepare statement: %w", err) } defer stmt.Close() - validOutputsStmt, err := ut.tx.Prepare(`INSERT INTO file_contract_valid_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) + validOutputsStmt, err := tx.Prepare(`INSERT INTO file_contract_valid_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContracts: failed to prepare valid proof outputs statement: %w", err) } defer validOutputsStmt.Close() - missedOutputsStmt, err := ut.tx.Prepare(`INSERT INTO file_contract_missed_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) + missedOutputsStmt, err := tx.Prepare(`INSERT INTO file_contract_missed_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContracts: failed to prepare missed proof outputs statement: %w", err) } @@ -170,20 +170,20 @@ func (ut *updateTx) addFileContracts(id int64, txn types.Transaction, fcDBIds ma return nil } -func (ut *updateTx) addFileContractRevisions(id int64, txn types.Transaction, dbIDs map[explorer.DBFileContract]int64) error { - stmt, err := ut.tx.Prepare(`INSERT INTO transaction_file_contract_revisions(transaction_id, transaction_order, contract_id, parent_id, unlock_conditions) VALUES (?, ?, ?, ?, ?)`) +func addFileContractRevisions(tx *txn, id int64, txn types.Transaction, dbIDs map[explorer.DBFileContract]int64) error { + stmt, err := tx.Prepare(`INSERT INTO transaction_file_contract_revisions(transaction_id, transaction_order, contract_id, parent_id, unlock_conditions) VALUES (?, ?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContractRevisions: failed to prepare statement: %w", err) } defer stmt.Close() - validOutputsStmt, err := ut.tx.Prepare(`INSERT INTO file_contract_valid_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) + validOutputsStmt, err := tx.Prepare(`INSERT INTO file_contract_valid_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContracts: failed to prepare valid proof outputs statement: %w", err) } defer validOutputsStmt.Close() - missedOutputsStmt, err := ut.tx.Prepare(`INSERT INTO file_contract_missed_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) + missedOutputsStmt, err := tx.Prepare(`INSERT INTO file_contract_missed_proof_outputs(contract_id, contract_order, address, value) VALUES (?, ?, ?, ?)`) if err != nil { return fmt.Errorf("addFileContracts: failed to prepare missed proof outputs statement: %w", err) } @@ -216,19 +216,19 @@ func (ut *updateTx) addFileContractRevisions(id int64, txn types.Transaction, db return nil } -func (ut *updateTx) addTransactions(bid types.BlockID, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[explorer.DBFileContract]int64) error { - checkTransactionStmt, err := ut.tx.Prepare(`SELECT id FROM transactions WHERE transaction_id = ?`) +func addTransactions(tx *txn, bid types.BlockID, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[explorer.DBFileContract]int64) error { + checkTransactionStmt, err := tx.Prepare(`SELECT id FROM transactions WHERE transaction_id = ?`) if err != nil { return fmt.Errorf("failed to prepare check transaction statement: %v", err) } - insertTransactionStmt, err := ut.tx.Prepare(`INSERT INTO transactions (transaction_id) VALUES (?)`) + insertTransactionStmt, err := tx.Prepare(`INSERT INTO transactions (transaction_id) VALUES (?)`) if err != nil { return fmt.Errorf("failed to prepare insert transaction statement: %v", err) } defer insertTransactionStmt.Close() - blockTransactionsStmt, err := ut.tx.Prepare(`INSERT INTO block_transactions(block_id, transaction_id, block_order) VALUES (?, ?, ?);`) + blockTransactionsStmt, err := tx.Prepare(`INSERT INTO block_transactions(block_id, transaction_id, block_order) VALUES (?, ?, ?);`) if err != nil { return fmt.Errorf("failed to prepare block_transactions statement: %w", err) } @@ -262,19 +262,19 @@ func (ut *updateTx) addTransactions(bid types.BlockID, txns []types.Transaction, if exist { continue } - if err := ut.addArbitraryData(txnID, txn); err != nil { + if err := addArbitraryData(tx, txnID, txn); err != nil { return fmt.Errorf("failed to add arbitrary data: %w", err) - } else if err := ut.addSiacoinInputs(txnID, txn); err != nil { + } else if err := addSiacoinInputs(tx, txnID, txn); err != nil { return fmt.Errorf("failed to add siacoin inputs: %w", err) - } else if err := ut.addSiacoinOutputs(txnID, txn, scDBIds); err != nil { + } else if err := addSiacoinOutputs(tx, txnID, txn, scDBIds); err != nil { return fmt.Errorf("failed to add siacoin outputs: %w", err) - } else if err := ut.addSiafundInputs(txnID, txn); err != nil { + } else if err := addSiafundInputs(tx, txnID, txn); err != nil { return fmt.Errorf("failed to add siafund inputs: %w", err) - } else if err := ut.addSiafundOutputs(txnID, txn, sfDBIds); err != nil { + } else if err := addSiafundOutputs(tx, txnID, txn, sfDBIds); err != nil { return fmt.Errorf("failed to add siafund outputs: %w", err) - } else if err := ut.addFileContracts(txnID, txn, fcDBIds); err != nil { + } else if err := addFileContracts(tx, txnID, txn, fcDBIds); err != nil { return fmt.Errorf("failed to add file contract: %w", err) - } else if err := ut.addFileContractRevisions(txnID, txn, fcDBIds); err != nil { + } else if err := addFileContractRevisions(tx, txnID, txn, fcDBIds); err != nil { return fmt.Errorf("failed to add file contract revisions: %w", err) } } @@ -287,7 +287,7 @@ type balance struct { sf uint64 } -func (ut *updateTx) updateBalances(height uint64, spentSiacoinElements, newSiacoinElements []explorer.SiacoinOutput, spentSiafundElements, newSiafundElements []types.SiafundElement) error { +func updateBalances(tx *txn, height uint64, spentSiacoinElements, newSiacoinElements []explorer.SiacoinOutput, spentSiafundElements, newSiafundElements []types.SiafundElement) error { addresses := make(map[types.Address]balance) for _, sce := range spentSiacoinElements { addresses[sce.SiacoinOutput.Address] = balance{} @@ -307,7 +307,7 @@ func (ut *updateTx) updateBalances(height uint64, spentSiacoinElements, newSiaco addressList = append(addressList, encode(address)) } - rows, err := ut.tx.Query(`SELECT address, siacoin_balance, immature_siacoin_balance, siafund_balance + rows, err := tx.Query(`SELECT address, siacoin_balance, immature_siacoin_balance, siafund_balance FROM address_balance WHERE address IN (`+queryPlaceHolders(len(addressList))+`)`, addressList...) if err != nil { @@ -357,7 +357,7 @@ func (ut *updateTx) updateBalances(height uint64, spentSiacoinElements, newSiaco addresses[sfe.SiafundOutput.Address] = bal } - stmt, err := ut.tx.Prepare(`INSERT INTO address_balance(address, siacoin_balance, immature_siacoin_balance, siafund_balance) + stmt, err := tx.Prepare(`INSERT INTO address_balance(address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES (?, ?, ?, ?) ON CONFLICT(address) DO UPDATE set siacoin_balance = ?, immature_siacoin_balance = ?, siafund_balance = ?`) @@ -376,14 +376,14 @@ func (ut *updateTx) updateBalances(height uint64, spentSiacoinElements, newSiaco return nil } -func (ut *updateTx) updateMaturedBalances(revert bool, height uint64) error { +func updateMaturedBalances(tx *txn, revert bool, height uint64) error { // Prevent double counting - outputs with a maturity height of 0 are // handled in updateBalances if height == 0 { return nil } - rows, err := ut.tx.Query(`SELECT address, value + rows, err := tx.Query(`SELECT address, value FROM siacoin_elements WHERE maturity_height = ?`, height) if err != nil { @@ -402,7 +402,7 @@ func (ut *updateTx) updateMaturedBalances(revert bool, height uint64) error { addressList = append(addressList, encode(sco.Address)) } - balanceRows, err := ut.tx.Query(`SELECT address, siacoin_balance, immature_siacoin_balance + balanceRows, err := tx.Query(`SELECT address, siacoin_balance, immature_siacoin_balance FROM address_balance WHERE address IN (`+queryPlaceHolders(len(addressList))+`)`, addressList...) if err != nil { @@ -434,7 +434,7 @@ func (ut *updateTx) updateMaturedBalances(revert bool, height uint64) error { addresses[sco.Address] = bal } - stmt, err := ut.tx.Prepare(`INSERT INTO address_balance(address, siacoin_balance, immature_siacoin_balance, siafund_balance) + stmt, err := tx.Prepare(`INSERT INTO address_balance(address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES (?, ?, ?, ?) ON CONFLICT(address) DO UPDATE set siacoin_balance = ?, immature_siacoin_balance = ?`) @@ -453,8 +453,8 @@ func (ut *updateTx) updateMaturedBalances(revert bool, height uint64) error { return nil } -func (ut *updateTx) updateStateTree(changes []explorer.TreeNodeUpdate) error { - stmt, err := ut.tx.Prepare(`INSERT INTO state_tree (row, column, value) VALUES($1, $2, $3) ON CONFLICT (row, column) DO UPDATE SET value=EXCLUDED.value;`) +func updateStateTree(tx *txn, changes []explorer.TreeNodeUpdate) error { + stmt, err := tx.Prepare(`INSERT INTO state_tree (row, column, value) VALUES($1, $2, $3) ON CONFLICT (row, column) DO UPDATE SET value=EXCLUDED.value;`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -469,8 +469,8 @@ func (ut *updateTx) updateStateTree(changes []explorer.TreeNodeUpdate) error { return nil } -func (ut *updateTx) addSiacoinElements(bid types.BlockID, spentElements, newElements []explorer.SiacoinOutput) (map[types.SiacoinOutputID]int64, error) { - stmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements(output_id, block_id, leaf_index, spent, source, maturity_height, address, value) +func addSiacoinElements(tx *txn, bid types.BlockID, spentElements, newElements []explorer.SiacoinOutput) (map[types.SiacoinOutputID]int64, error) { + stmt, err := tx.Prepare(`INSERT INTO siacoin_elements(output_id, block_id, leaf_index, spent, source, maturity_height, address, value) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (output_id) DO UPDATE SET spent = ?, leaf_index = ?`) @@ -510,8 +510,8 @@ func (ut *updateTx) addSiacoinElements(bid types.BlockID, spentElements, newElem return scDBIds, nil } -func (ut *updateTx) addSiafundElements(bid types.BlockID, spentElements, newElements []types.SiafundElement) (map[types.SiafundOutputID]int64, error) { - stmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements(output_id, block_id, leaf_index, spent, claim_start, address, value) +func addSiafundElements(tx *txn, bid types.BlockID, spentElements, newElements []types.SiafundElement) (map[types.SiafundOutputID]int64, error) { + stmt, err := tx.Prepare(`INSERT INTO siafund_elements(output_id, block_id, leaf_index, spent, claim_start, address, value) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO UPDATE SET spent = ?, leaf_index = ?`) @@ -551,13 +551,13 @@ func (ut *updateTx) addSiafundElements(bid types.BlockID, spentElements, newElem return sfDBIds, nil } -func (ut *updateTx) deleteBlock(bid types.BlockID) error { - _, err := ut.tx.Exec("DELETE FROM blocks WHERE id = ?", encode(bid)) +func deleteBlock(tx *txn, bid types.BlockID) error { + _, err := tx.Exec("DELETE FROM blocks WHERE id = ?", encode(bid)) return err } -func (ut *updateTx) addFileContractElements(bid types.BlockID, fces []explorer.FileContractUpdate) (map[explorer.DBFileContract]int64, error) { - stmt, err := ut.tx.Prepare(`INSERT INTO file_contract_elements(block_id, contract_id, leaf_index, resolved, valid, filesize, file_merkle_root, window_start, window_end, payout, unlock_hash, revision_number) +func addFileContractElements(tx *txn, bid types.BlockID, fces []explorer.FileContractUpdate) (map[explorer.DBFileContract]int64, error) { + stmt, err := tx.Prepare(`INSERT INTO file_contract_elements(block_id, contract_id, leaf_index, resolved, valid, filesize, file_merkle_root, window_start, window_end, payout, unlock_hash, revision_number) VALUES (?, ?, ?, FALSE, TRUE, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (contract_id, revision_number) DO UPDATE SET resolved = ?, valid = ?, leaf_index = ? @@ -567,7 +567,7 @@ func (ut *updateTx) addFileContractElements(bid types.BlockID, fces []explorer.F } defer stmt.Close() - revisionStmt, err := ut.tx.Prepare(`INSERT INTO last_contract_revision(contract_id, contract_element_id) + revisionStmt, err := tx.Prepare(`INSERT INTO last_contract_revision(contract_id, contract_element_id) VALUES (?, ?) ON CONFLICT (contract_id) DO UPDATE SET contract_element_id = ?`) @@ -601,13 +601,14 @@ func (ut *updateTx) addFileContractElements(bid types.BlockID, fces []explorer.F } func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { - if err := ut.addBlock(state.Block, state.Index.Height); err != nil { + if err := addBlock(ut.tx, state.Block, state.Index.Height); err != nil { return fmt.Errorf("ApplyIndex: failed to add block: %w", err) - } else if err := ut.updateMaturedBalances(false, state.Index.Height); err != nil { + } else if err := updateMaturedBalances(ut.tx, false, state.Index.Height); err != nil { return fmt.Errorf("ApplyIndex: failed to update matured balances: %w", err) } - scDBIds, err := ut.addSiacoinElements( + scDBIds, err := addSiacoinElements( + ut.tx, state.Block.ID(), append(state.SpentSiacoinElements, state.EphemeralSiacoinElements...), state.NewSiacoinElements, @@ -615,7 +616,8 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { if err != nil { return fmt.Errorf("ApplyIndex: failed to add siacoin outputs: %w", err) } - sfDBIds, err := ut.addSiafundElements( + sfDBIds, err := addSiafundElements( + ut.tx, state.Block.ID(), append(state.SpentSiafundElements, state.EphemeralSiafundElements...), state.NewSiafundElements, @@ -623,20 +625,20 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { if err != nil { return fmt.Errorf("ApplyIndex: failed to add siafund outputs: %w", err) } - if err := ut.updateBalances(state.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { + if err := updateBalances(ut.tx, state.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { return fmt.Errorf("ApplyIndex: failed to update balances: %w", err) } - fcDBIds, err := ut.addFileContractElements(state.Block.ID(), state.FileContractElements) + fcDBIds, err := addFileContractElements(ut.tx, state.Block.ID(), state.FileContractElements) if err != nil { return fmt.Errorf("v: failed to add file contracts: %w", err) } - if err := ut.addMinerPayouts(state.Block.ID(), state.Block.MinerPayouts, scDBIds); err != nil { + if err := addMinerPayouts(ut.tx, state.Block.ID(), state.Block.MinerPayouts, scDBIds); err != nil { return fmt.Errorf("ApplyIndex: failed to add miner payouts: %w", err) - } else if err := ut.addTransactions(state.Block.ID(), state.Block.Transactions, scDBIds, sfDBIds, fcDBIds); err != nil { + } else if err := addTransactions(ut.tx, state.Block.ID(), state.Block.Transactions, scDBIds, sfDBIds, fcDBIds); err != nil { return fmt.Errorf("ApplyIndex: failed to add transactions: addTransactions: %w", err) - } else if err := ut.updateStateTree(state.TreeUpdates); err != nil { + } else if err := updateStateTree(ut.tx, state.TreeUpdates); err != nil { return fmt.Errorf("ApplyIndex: failed to update state tree: %w", err) } @@ -644,27 +646,29 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { } func (ut *updateTx) RevertIndex(state explorer.UpdateState) error { - if err := ut.updateMaturedBalances(true, state.Index.Height); err != nil { + if err := updateMaturedBalances(ut.tx, true, state.Index.Height); err != nil { return fmt.Errorf("RevertIndex: failed to update matured balances: %w", err) - } else if _, err := ut.addSiacoinElements( + } else if _, err := addSiacoinElements( + ut.tx, state.Block.ID(), state.SpentSiacoinElements, append(state.NewSiacoinElements, state.EphemeralSiacoinElements...), ); err != nil { return fmt.Errorf("RevertIndex: failed to update siacoin output state: %w", err) - } else if _, err := ut.addSiafundElements( + } else if _, err := addSiafundElements( + ut.tx, state.Block.ID(), state.SpentSiafundElements, append(state.NewSiafundElements, state.EphemeralSiafundElements...), ); err != nil { return fmt.Errorf("RevertIndex: failed to update siafund output state: %w", err) - } else if err := ut.updateBalances(state.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { + } else if err := updateBalances(ut.tx, state.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { return fmt.Errorf("RevertIndex: failed to update balances: %w", err) - } else if _, err := ut.addFileContractElements(state.Block.ID(), state.FileContractElements); err != nil { + } else if _, err := addFileContractElements(ut.tx, state.Block.ID(), state.FileContractElements); err != nil { return fmt.Errorf("RevertIndex: failed to update file contract state: %w", err) - } else if err := ut.deleteBlock(state.Block.ID()); err != nil { + } else if err := deleteBlock(ut.tx, state.Block.ID()); err != nil { return fmt.Errorf("RevertIndex: failed to delete block: %w", err) - } else if err := ut.updateStateTree(state.TreeUpdates); err != nil { + } else if err := updateStateTree(ut.tx, state.TreeUpdates); err != nil { return fmt.Errorf("RevertIndex: failed to update state tree: %w", err) }