From 23c759aef23b3c029a22f5058ecc477cd80dfcc5 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Tue, 8 Oct 2024 00:16:40 -0400 Subject: [PATCH 1/5] store transaction_id in file_contract_elements --- persist/sqlite/consensus.go | 24 +++++++++++++++++++++--- persist/sqlite/init.sql | 1 + 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 961bd545..9449e304 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -713,8 +713,8 @@ func deleteBlock(tx *txn, bid types.BlockID) error { } func updateFileContractElements(tx *txn, revert bool, b types.Block, fces []explorer.FileContractUpdate) (map[explorer.DBFileContract]int64, error) { - stmt, err := tx.Prepare(`INSERT INTO file_contract_elements(contract_id, block_id, leaf_index, resolved, valid, filesize, file_merkle_root, window_start, window_end, payout, unlock_hash, revision_number) - VALUES (?, ?, ?, FALSE, FALSE, ?, ?, ?, ?, ?, ?, ?) + stmt, err := tx.Prepare(`INSERT INTO file_contract_elements(contract_id, block_id, transaction_id, leaf_index, resolved, valid, filesize, file_merkle_root, window_start, window_end, payout, unlock_hash, revision_number) + VALUES (?, ?, ?, ?, FALSE, FALSE, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (contract_id, revision_number) DO UPDATE SET resolved = ?, valid = ?, leaf_index = ? RETURNING id;`) @@ -775,11 +775,29 @@ func updateFileContractElements(tx *txn, revert bool, b types.Block, fces []expl } } + fcTxns := make(map[explorer.DBFileContract]types.TransactionID) + for _, txn := range b.Transactions { + id := txn.ID() + + for i, fc := range txn.FileContracts { + fcTxns[explorer.DBFileContract{ + ID: txn.FileContractID(i), + RevisionNumber: fc.RevisionNumber, + }] = id + } + for _, fcr := range txn.FileContractRevisions { + fcTxns[explorer.DBFileContract{ + ID: fcr.ParentID, + RevisionNumber: fcr.FileContract.RevisionNumber, + }] = id + } + } + fcDBIds := make(map[explorer.DBFileContract]int64) addFC := func(fcID types.FileContractID, leafIndex uint64, fc types.FileContract, resolved, valid, lastRevision bool) error { var dbID int64 dbFC := explorer.DBFileContract{ID: fcID, RevisionNumber: fc.RevisionNumber} - err := stmt.QueryRow(encode(fcID), encode(b.ID()), encode(leafIndex), encode(fc.Filesize), encode(fc.FileMerkleRoot), encode(fc.WindowStart), encode(fc.WindowEnd), encode(fc.Payout), encode(fc.UnlockHash), encode(fc.RevisionNumber), resolved, valid, encode(leafIndex)).Scan(&dbID) + err := stmt.QueryRow(encode(fcID), encode(b.ID()), encode(fcTxns[dbFC]), encode(leafIndex), encode(fc.Filesize), encode(fc.FileMerkleRoot), encode(fc.WindowStart), encode(fc.WindowEnd), encode(fc.Payout), encode(fc.UnlockHash), encode(fc.RevisionNumber), resolved, valid, encode(leafIndex)).Scan(&dbID) if err != nil { return fmt.Errorf("failed to execute file_contract_elements statement: %w", err) } diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index b34ae8d8..2d9a3324 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -78,6 +78,7 @@ CREATE TABLE file_contract_elements ( block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, contract_id BLOB NOT NULL, + transaction_id BLOB NOT NULL, leaf_index BLOB NOT NULL, resolved INTEGER NOT NULL, From 4febc7884cbb178c1951649db56c9bf66712c7d8 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Tue, 8 Oct 2024 00:43:52 -0400 Subject: [PATCH 2/5] retrieve transaction ID --- explorer/types.go | 2 ++ persist/sqlite/consensus_test.go | 14 ++++++++++++++ persist/sqlite/contracts.go | 8 ++++---- persist/sqlite/init.sql | 3 ++- persist/sqlite/transactions.go | 8 ++++---- 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/explorer/types.go b/explorer/types.go index 113f0ecb..820bb763 100644 --- a/explorer/types.go +++ b/explorer/types.go @@ -129,6 +129,8 @@ type FileContract struct { Resolved bool `json:"resolved"` Valid bool `json:"valid"` + TransactionID types.TransactionID `json:"transactionID"` + ConfirmationIndex *types.ChainIndex `json:"confirmationIndex"` ConfirmationTransactionID *types.TransactionID `json:"confirmationTransactionID"` diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index 507a4c64..485fd494 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -584,6 +584,7 @@ func TestFileContract(t *testing.T) { } testutil.Equal(t, "fcs", 1, len(dbFCs)) testutil.CheckFC(t, false, false, false, fc, dbFCs[0]) + testutil.Equal(t, "transaction ID", txn.ID(), dbFCs[0].TransactionID) testutil.Equal(t, "confirmation index", cm.Tip(), *dbFCs[0].ConfirmationIndex) testutil.Equal(t, "confirmation transaction ID", txn.ID(), *dbFCs[0].ConfirmationTransactionID) } @@ -605,6 +606,7 @@ func TestFileContract(t *testing.T) { testutil.Equal(t, "file contracts", 1, len(txns[0].FileContracts)) testutil.CheckFC(t, false, false, false, fc, txns[0].FileContracts[0]) + testutil.Equal(t, "transaction ID", txn.ID(), txns[0].FileContracts[0].TransactionID) testutil.Equal(t, "confirmation index", cm.Tip(), *txns[0].FileContracts[0].ConfirmationIndex) testutil.Equal(t, "confirmation transaction ID", txn.ID(), *txns[0].FileContracts[0].ConfirmationTransactionID) } @@ -646,6 +648,8 @@ func TestFileContract(t *testing.T) { testutil.CheckFC(t, false, false, false, fc, renterContracts[0]) testutil.CheckFC(t, false, false, false, fc, hostContracts[0]) + testutil.Equal(t, "transaction ID", reviseTxn.ID(), renterContracts[0].TransactionID) + testutil.Equal(t, "transaction ID", reviseTxn.ID(), hostContracts[0].TransactionID) testutil.Equal(t, "confirmation index", prevTip, *renterContracts[0].ConfirmationIndex) testutil.Equal(t, "confirmation transaction ID", txn.ID(), *renterContracts[0].ConfirmationTransactionID) testutil.Equal(t, "confirmation index", prevTip, *hostContracts[0].ConfirmationIndex) @@ -748,6 +752,8 @@ func TestFileContract(t *testing.T) { testutil.CheckFC(t, false, true, false, fc, renterContracts[0]) testutil.CheckFC(t, false, true, false, fc, hostContracts[0]) + testutil.Equal(t, "transaction ID", reviseTxn.ID(), renterContracts[0].TransactionID) + testutil.Equal(t, "transaction ID", reviseTxn.ID(), hostContracts[0].TransactionID) testutil.Equal(t, "confirmation index", prevTip, *renterContracts[0].ConfirmationIndex) testutil.Equal(t, "confirmation transaction ID", txn.ID(), *renterContracts[0].ConfirmationTransactionID) testutil.Equal(t, "confirmation index", prevTip, *hostContracts[0].ConfirmationIndex) @@ -875,6 +881,7 @@ func TestEphemeralFileContract(t *testing.T) { } testutil.Equal(t, "fcs", 1, len(dbFCs)) testutil.CheckFC(t, true, false, false, revisedFC1, dbFCs[0]) + testutil.Equal(t, "transaction ID", reviseTxn1.ID(), dbFCs[0].TransactionID) } { @@ -894,6 +901,7 @@ func TestEphemeralFileContract(t *testing.T) { testutil.Equal(t, "file contracts", 1, len(txns[0].FileContracts)) testutil.CheckFC(t, true, false, false, fc, txns[0].FileContracts[0]) + testutil.Equal(t, "transaction ID", txn.ID(), txns[0].FileContracts[0].TransactionID) testutil.Equal(t, "confirmation index", cm.Tip(), *txns[0].FileContracts[0].ConfirmationIndex) testutil.Equal(t, "confirmation transaction ID", txn.ID(), *txns[0].FileContracts[0].ConfirmationTransactionID) } @@ -955,6 +963,7 @@ func TestEphemeralFileContract(t *testing.T) { } testutil.Equal(t, "fcs", 1, len(dbFCs)) testutil.CheckFC(t, true, false, false, revisedFC3, dbFCs[0]) + testutil.Equal(t, "transaction ID", reviseTxn3.ID(), dbFCs[0].TransactionID) } { @@ -2302,6 +2311,7 @@ func TestMultipleReorgFileContract(t *testing.T) { testutil.Equal(t, "fcs", 1, len(dbFCs)) testutil.CheckFC(t, false, false, false, revFC, dbFCs[0]) + testutil.Equal(t, "transaction ID", reviseTxn.ID(), dbFCs[0].TransactionID) testutil.Equal(t, "confirmation index", prevState1.Index, *dbFCs[0].ConfirmationIndex) testutil.Equal(t, "confirmation transaction ID", txn.ID(), *dbFCs[0].ConfirmationTransactionID) } @@ -2340,6 +2350,8 @@ func TestMultipleReorgFileContract(t *testing.T) { } testutil.Equal(t, "renter contracts and host contracts", len(renterContracts), len(hostContracts)) testutil.Equal(t, "len(contracts)", 1, len(renterContracts)) + testutil.Equal(t, "transaction ID", reviseTxn.ID(), renterContracts[0].TransactionID) + testutil.Equal(t, "transaction ID", reviseTxn.ID(), hostContracts[0].TransactionID) testutil.CheckFC(t, false, false, false, revFC, renterContracts[0]) testutil.CheckFC(t, false, false, false, revFC, hostContracts[0]) } @@ -2373,6 +2385,7 @@ func TestMultipleReorgFileContract(t *testing.T) { testutil.Equal(t, "fcs", 1, len(dbFCs)) testutil.CheckFC(t, false, false, false, fc, dbFCs[0]) + testutil.Equal(t, "transaction ID", txn.ID(), dbFCs[0].TransactionID) testutil.Equal(t, "confirmation index", prevState1.Index, *dbFCs[0].ConfirmationIndex) testutil.Equal(t, "confirmation transaction ID", txn.ID(), *dbFCs[0].ConfirmationTransactionID) } @@ -2423,6 +2436,7 @@ func TestMultipleReorgFileContract(t *testing.T) { testutil.Equal(t, "fcs", 1, len(dbFCs)) testutil.CheckFC(t, false, false, false, revFC, dbFCs[0]) + testutil.Equal(t, "transaction ID", reviseTxn.ID(), dbFCs[0].TransactionID) testutil.Equal(t, "confirmation index", prevState1.Index, *dbFCs[0].ConfirmationIndex) testutil.Equal(t, "confirmation transaction ID", txn.ID(), *dbFCs[0].ConfirmationTransactionID) } diff --git a/persist/sqlite/contracts.go b/persist/sqlite/contracts.go index c9975753..8e744072 100644 --- a/persist/sqlite/contracts.go +++ b/persist/sqlite/contracts.go @@ -18,7 +18,7 @@ func encodedIDs(ids []types.FileContractID) []any { func scanFileContract(s scanner) (contractID int64, fc explorer.FileContract, err error) { var confirmationIndex, proofIndex types.ChainIndex var confirmationTransactionID, proofTransactionID types.TransactionID - err = s.Scan(&contractID, decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), &fc.Resolved, &fc.Valid, decodeNull(&confirmationIndex), decodeNull(&confirmationTransactionID), decodeNull(&proofIndex), decodeNull(&proofTransactionID), decode(&fc.FileContract.Filesize), decode(&fc.FileContract.FileMerkleRoot), decode(&fc.FileContract.WindowStart), decode(&fc.FileContract.WindowEnd), decode(&fc.FileContract.Payout), decode(&fc.FileContract.UnlockHash), decode(&fc.FileContract.RevisionNumber)) + err = s.Scan(&contractID, decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), &fc.Resolved, &fc.Valid, decode(&fc.TransactionID), decodeNull(&confirmationIndex), decodeNull(&confirmationTransactionID), decodeNull(&proofIndex), decodeNull(&proofTransactionID), decode(&fc.FileContract.Filesize), decode(&fc.FileContract.FileMerkleRoot), decode(&fc.FileContract.WindowStart), decode(&fc.FileContract.WindowEnd), decode(&fc.FileContract.Payout), decode(&fc.FileContract.UnlockHash), decode(&fc.FileContract.RevisionNumber)) if confirmationIndex != (types.ChainIndex{}) { fc.ConfirmationIndex = &confirmationIndex @@ -39,7 +39,7 @@ func scanFileContract(s scanner) (contractID int64, fc explorer.FileContract, er // Contracts implements explorer.Store. func (s *Store) Contracts(ids []types.FileContractID) (result []explorer.FileContract, err error) { err = s.transaction(func(tx *txn) error { - query := `SELECT fc1.id, fc1.contract_id, fc1.leaf_index, fc1.resolved, fc1.valid, rev.confirmation_index, rev.confirmation_transaction_id, rev.proof_index, rev.proof_transaction_id, fc1.filesize, fc1.file_merkle_root, fc1.window_start, fc1.window_end, fc1.payout, fc1.unlock_hash, fc1.revision_number + query := `SELECT fc1.id, fc1.contract_id, fc1.leaf_index, fc1.resolved, fc1.valid, fc1.transaction_id, rev.confirmation_index, rev.confirmation_transaction_id, rev.proof_index, rev.proof_transaction_id, fc1.filesize, fc1.file_merkle_root, fc1.window_start, fc1.window_end, fc1.payout, fc1.unlock_hash, fc1.revision_number FROM file_contract_elements fc1 INNER JOIN last_contract_revision rev ON (rev.contract_element_id = fc1.id) WHERE rev.contract_id IN (` + queryPlaceHolders(len(ids)) + `)` @@ -84,7 +84,7 @@ func (s *Store) Contracts(ids []types.FileContractID) (result []explorer.FileCon // ContractRevisions implements explorer.Store. func (s *Store) ContractRevisions(id types.FileContractID) (revisions []explorer.FileContract, err error) { err = s.transaction(func(tx *txn) error { - query := `SELECT fc.id, fc.contract_id, fc.leaf_index, fc.resolved, fc.valid, rev.confirmation_index, rev.confirmation_transaction_id, rev.proof_index, rev.proof_transaction_id, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number + query := `SELECT fc.id, fc.contract_id, fc.leaf_index, fc.resolved, fc.valid, fc.transaction_id, rev.confirmation_index, rev.confirmation_transaction_id, rev.proof_index, rev.proof_transaction_id, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number FROM file_contract_elements fc JOIN last_contract_revision rev ON (rev.contract_id = fc.contract_id) WHERE fc.contract_id = ? @@ -145,7 +145,7 @@ func (s *Store) ContractRevisions(id types.FileContractID) (revisions []explorer // ContractsKey implements explorer.Store. func (s *Store) ContractsKey(key types.PublicKey) (result []explorer.FileContract, err error) { err = s.transaction(func(tx *txn) error { - query := `SELECT fc1.id, fc1.contract_id, fc1.leaf_index, fc1.resolved, fc1.valid, rev.confirmation_index, rev.confirmation_transaction_id, rev.proof_index, rev.proof_transaction_id, fc1.filesize, fc1.file_merkle_root, fc1.window_start, fc1.window_end, fc1.payout, fc1.unlock_hash, fc1.revision_number + query := `SELECT fc1.id, fc1.contract_id, fc1.leaf_index, fc1.resolved, fc1.valid, fc1.transaction_id, rev.confirmation_index, rev.confirmation_transaction_id, rev.proof_index, rev.proof_transaction_id, fc1.filesize, fc1.file_merkle_root, fc1.window_start, fc1.window_end, fc1.payout, fc1.unlock_hash, fc1.revision_number FROM file_contract_elements fc1 INNER JOIN last_contract_revision rev ON (rev.contract_element_id = fc1.id) WHERE rev.ed25519_renter_key = ? OR rev.ed25519_host_key = ?` diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 2d9a3324..8a244990 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -76,9 +76,10 @@ CREATE INDEX siafund_elements_address_spent_index ON siafund_elements(address, s CREATE TABLE file_contract_elements ( id INTEGER PRIMARY KEY, block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, + -- We need to defer because transactions are inserted after file contracts + transaction_id BLOB NOT NULL REFERENCES transactions(transaction_id) ON DELETE CASCADE NOT NULL DEFERRABLE INITIALLY DEFERRED, contract_id BLOB NOT NULL, - transaction_id BLOB NOT NULL, leaf_index BLOB NOT NULL, resolved INTEGER NOT NULL, diff --git a/persist/sqlite/transactions.go b/persist/sqlite/transactions.go index 7d73c017..412ed41c 100644 --- a/persist/sqlite/transactions.go +++ b/persist/sqlite/transactions.go @@ -282,7 +282,7 @@ type contractOrder struct { // transactionFileContracts returns the file contracts for each transaction. func transactionFileContracts(tx *txn, txnIDs []int64) (map[int64][]explorer.FileContract, error) { - query := `SELECT ts.transaction_id, fc.id, rev.confirmation_index, rev.confirmation_transaction_id, rev.proof_index, rev.proof_transaction_id, fc.contract_id, fc.leaf_index, fc.resolved, fc.valid, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number + query := `SELECT ts.transaction_id, fc.id, rev.confirmation_index, rev.confirmation_transaction_id, rev.proof_index, rev.proof_transaction_id, fc.contract_id, fc.leaf_index, fc.resolved, fc.valid, fc.transaction_id, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number FROM file_contract_elements fc INNER JOIN transaction_file_contracts ts ON (ts.contract_id = fc.id) INNER JOIN last_contract_revision rev ON (rev.contract_id = fc.contract_id) @@ -305,7 +305,7 @@ ORDER BY ts.transaction_order ASC` var confirmationIndex, proofIndex types.ChainIndex var confirmationTransactionID, proofTransactionID types.TransactionID - if err := rows.Scan(&txnID, &contractID, decodeNull(&confirmationIndex), decodeNull(&confirmationTransactionID), decodeNull(&proofIndex), decodeNull(&proofTransactionID), decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), &fc.Resolved, &fc.Valid, decode(&fc.FileContract.Filesize), decode(&fc.FileContract.FileMerkleRoot), decode(&fc.FileContract.WindowStart), decode(&fc.FileContract.WindowEnd), decode(&fc.FileContract.Payout), decode(&fc.FileContract.UnlockHash), decode(&fc.FileContract.RevisionNumber)); err != nil { + if err := rows.Scan(&txnID, &contractID, decodeNull(&confirmationIndex), decodeNull(&confirmationTransactionID), decodeNull(&proofIndex), decodeNull(&proofTransactionID), decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), &fc.Resolved, &fc.Valid, decode(&fc.TransactionID), decode(&fc.FileContract.Filesize), decode(&fc.FileContract.FileMerkleRoot), decode(&fc.FileContract.WindowStart), decode(&fc.FileContract.WindowEnd), decode(&fc.FileContract.Payout), decode(&fc.FileContract.UnlockHash), decode(&fc.FileContract.RevisionNumber)); err != nil { return nil, fmt.Errorf("failed to scan file contract: %w", err) } @@ -343,7 +343,7 @@ ORDER BY ts.transaction_order ASC` // transactionFileContracts returns the file contract revisions for each transaction. func transactionFileContractRevisions(tx *txn, txnIDs []int64) (map[int64][]explorer.FileContractRevision, error) { - query := `SELECT ts.transaction_id, fc.id, rev.confirmation_index, rev.confirmation_transaction_id, rev.proof_index, rev.proof_transaction_id, ts.parent_id, ts.unlock_conditions, fc.contract_id, fc.leaf_index, fc.resolved, fc.valid, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number + query := `SELECT ts.transaction_id, fc.id, rev.confirmation_index, rev.confirmation_transaction_id, rev.proof_index, rev.proof_transaction_id, ts.parent_id, ts.unlock_conditions, fc.contract_id, fc.leaf_index, fc.resolved, fc.valid, fc.transaction_id, fc.filesize, fc.file_merkle_root, fc.window_start, fc.window_end, fc.payout, fc.unlock_hash, fc.revision_number FROM file_contract_elements fc INNER JOIN transaction_file_contract_revisions ts ON (ts.contract_id = fc.id) INNER JOIN last_contract_revision rev ON (rev.contract_id = fc.contract_id) @@ -366,7 +366,7 @@ ORDER BY ts.transaction_order ASC` var confirmationIndex, proofIndex types.ChainIndex var confirmationTransactionID, proofTransactionID types.TransactionID - if err := rows.Scan(&txnID, &contractID, decodeNull(&confirmationIndex), decodeNull(&confirmationTransactionID), decodeNull(&proofIndex), decodeNull(&proofTransactionID), decode(&fc.ParentID), decode(&fc.UnlockConditions), decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), &fc.Resolved, &fc.Valid, decode(&fc.FileContract.FileContract.Filesize), decode(&fc.FileContract.FileContract.FileMerkleRoot), decode(&fc.FileContract.FileContract.WindowStart), decode(&fc.FileContract.FileContract.WindowEnd), decode(&fc.FileContract.FileContract.Payout), decode(&fc.FileContract.FileContract.UnlockHash), decode(&fc.FileContract.FileContract.RevisionNumber)); err != nil { + if err := rows.Scan(&txnID, &contractID, decodeNull(&confirmationIndex), decodeNull(&confirmationTransactionID), decodeNull(&proofIndex), decodeNull(&proofTransactionID), decode(&fc.ParentID), decode(&fc.UnlockConditions), decode(&fc.StateElement.ID), decode(&fc.StateElement.LeafIndex), &fc.Resolved, &fc.Valid, decode(&fc.TransactionID), decode(&fc.FileContract.FileContract.Filesize), decode(&fc.FileContract.FileContract.FileMerkleRoot), decode(&fc.FileContract.FileContract.WindowStart), decode(&fc.FileContract.FileContract.WindowEnd), decode(&fc.FileContract.FileContract.Payout), decode(&fc.FileContract.FileContract.UnlockHash), decode(&fc.FileContract.FileContract.RevisionNumber)); err != nil { return nil, fmt.Errorf("failed to scan file contract: %w", err) } From e92cce6584f256fa96aa2b12176bcc5720250fdf Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Wed, 9 Oct 2024 14:17:38 -0400 Subject: [PATCH 3/5] index file contract elements on contract_id and revision_number --- persist/sqlite/init.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 8a244990..7b909692 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -94,7 +94,7 @@ CREATE TABLE file_contract_elements ( revision_number BLOB NOT NULL, UNIQUE(contract_id, revision_number) ); -CREATE INDEX file_contract_elements_contract_id_index ON file_contract_elements(contract_id); +CREATE INDEX file_contract_elements_contract_id_revision_number,index ON file_contract_elements(contract_id, revision_number); CREATE TABLE last_contract_revision ( contract_id BLOB PRIMARY KEY NOT NULL, From 92232095ad78adf9264e173c54e872e7b36d95a8 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Wed, 9 Oct 2024 15:02:31 -0400 Subject: [PATCH 4/5] remove need to defer foreign key checks by inserting into transactions table first and then inserting actual transaction data afterwards --- persist/sqlite/consensus.go | 88 +++++++++++++++++++++++-------------- persist/sqlite/init.sql | 5 +-- 2 files changed, 56 insertions(+), 37 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 9449e304..4d948b5f 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -215,7 +215,12 @@ func addStorageProofs(tx *txn, id int64, txn types.Transaction) error { return nil } -func addTransactions(tx *txn, bid types.BlockID, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[explorer.DBFileContract]int64) (map[types.TransactionID]int64, error) { +type txnDBId struct { + id int64 + exist bool +} + +func addTransactions(tx *txn, bid types.BlockID, txns []types.Transaction) (map[types.TransactionID]txnDBId, error) { checkTransactionStmt, err := tx.Prepare(`SELECT id FROM transactions WHERE transaction_id = ?`) if err != nil { return nil, fmt.Errorf("failed to prepare check transaction statement: %v", err) @@ -234,7 +239,7 @@ func addTransactions(tx *txn, bid types.BlockID, txns []types.Transaction, scDBI } defer blockTransactionsStmt.Close() - txnDBIds := make(map[types.TransactionID]int64) + txnDBIds := make(map[types.TransactionID]txnDBId) for i, txn := range txns { var exist bool var txnID int64 @@ -254,40 +259,52 @@ func addTransactions(tx *txn, bid types.BlockID, txns []types.Transaction, scDBI return nil, fmt.Errorf("failed to get transaction ID: %w", err) } } - txnDBIds[txn.ID()] = txnID + txnDBIds[txn.ID()] = txnDBId{id: txnID, exist: exist} if _, err := blockTransactionsStmt.Exec(encode(bid), txnID, i); err != nil { return nil, fmt.Errorf("failed to insert into block_transactions: %w", err) } + } + return txnDBIds, nil +} + +func addTransactionFields(tx *txn, txns []types.Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[explorer.DBFileContract]int64, txnDBIds map[types.TransactionID]txnDBId) error { + for _, txn := range txns { + dbID, ok := txnDBIds[txn.ID()] + if !ok { + panic(fmt.Errorf("txn %v should be in txnDBIds", txn.ID())) + } + // transaction already exists, don't reinsert its fields - if exist { + if dbID.exist { continue } - if err := addMinerFees(tx, txnID, txn); err != nil { - return nil, fmt.Errorf("failed to add miner fees: %w", err) - } else if err := addArbitraryData(tx, txnID, txn); err != nil { - return nil, fmt.Errorf("failed to add arbitrary data: %w", err) - } else if err := addSignatures(tx, txnID, txn); err != nil { - return nil, fmt.Errorf("failed to add signatures: %w", err) - } else if err := addSiacoinInputs(tx, txnID, txn); err != nil { - return nil, fmt.Errorf("failed to add siacoin inputs: %w", err) - } else if err := addSiacoinOutputs(tx, txnID, txn, scDBIds); err != nil { - return nil, fmt.Errorf("failed to add siacoin outputs: %w", err) - } else if err := addSiafundInputs(tx, txnID, txn); err != nil { - return nil, fmt.Errorf("failed to add siafund inputs: %w", err) - } else if err := addSiafundOutputs(tx, txnID, txn, sfDBIds); err != nil { - return nil, fmt.Errorf("failed to add siafund outputs: %w", err) - } else if err := addFileContracts(tx, txnID, txn, fcDBIds); err != nil { - return nil, fmt.Errorf("failed to add file contract: %w", err) - } else if err := addFileContractRevisions(tx, txnID, txn, fcDBIds); err != nil { - return nil, fmt.Errorf("failed to add file contract revisions: %w", err) - } else if err := addStorageProofs(tx, txnID, txn); err != nil { - return nil, fmt.Errorf("failed to add storage proofs: %w", err) + if err := addMinerFees(tx, dbID.id, txn); err != nil { + return fmt.Errorf("failed to add miner fees: %w", err) + } else if err := addArbitraryData(tx, dbID.id, txn); err != nil { + return fmt.Errorf("failed to add arbitrary data: %w", err) + } else if err := addSignatures(tx, dbID.id, txn); err != nil { + return fmt.Errorf("failed to add signatures: %w", err) + } else if err := addSiacoinInputs(tx, dbID.id, txn); err != nil { + return fmt.Errorf("failed to add siacoin inputs: %w", err) + } else if err := addSiacoinOutputs(tx, dbID.id, txn, scDBIds); err != nil { + return fmt.Errorf("failed to add siacoin outputs: %w", err) + } else if err := addSiafundInputs(tx, dbID.id, txn); err != nil { + return fmt.Errorf("failed to add siafund inputs: %w", err) + } else if err := addSiafundOutputs(tx, dbID.id, txn, sfDBIds); err != nil { + return fmt.Errorf("failed to add siafund outputs: %w", err) + } else if err := addFileContracts(tx, dbID.id, txn, fcDBIds); err != nil { + return fmt.Errorf("failed to add file contract: %w", err) + } else if err := addFileContractRevisions(tx, dbID.id, txn, fcDBIds); err != nil { + return fmt.Errorf("failed to add file contract revisions: %w", err) + } else if err := addStorageProofs(tx, dbID.id, txn); err != nil { + return fmt.Errorf("failed to add storage proofs: %w", err) } } - return txnDBIds, nil + + return nil } type balance struct { @@ -578,7 +595,7 @@ func addSiafundElements(tx *txn, index types.ChainIndex, spentElements, newEleme return sfDBIds, nil } -func addEvents(tx *txn, scDBIds map[types.SiacoinOutputID]int64, fcDBIds map[explorer.DBFileContract]int64, txnDBIds map[types.TransactionID]int64, events []explorer.Event) error { +func addEvents(tx *txn, scDBIds map[types.SiacoinOutputID]int64, fcDBIds map[explorer.DBFileContract]int64, txnDBIds map[types.TransactionID]txnDBId, events []explorer.Event) error { if len(events) == 0 { return nil } @@ -649,7 +666,7 @@ func addEvents(tx *txn, scDBIds map[types.SiacoinOutputID]int64, fcDBIds map[exp switch v := event.Data.(type) { case *explorer.EventTransaction: - dbID := txnDBIds[types.TransactionID(event.ID)] + dbID := txnDBIds[types.TransactionID(event.ID)].id if _, err = transactionEventStmt.Exec(eventID, dbID, encode(v.Fee)); err != nil { return fmt.Errorf("failed to insert transaction event: %w", err) } @@ -984,6 +1001,11 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { return fmt.Errorf("ApplyIndex: failed to update matured balances: %w", err) } + txnDBIds, err := addTransactions(ut.tx, state.Block.ID(), state.Block.Transactions) + if err != nil { + return fmt.Errorf("ApplyIndex: failed to add transactions: %w", err) + } + scDBIds, err := addSiacoinElements( ut.tx, state.Metrics.Index, @@ -1002,19 +1024,17 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { if err != nil { return fmt.Errorf("ApplyIndex: failed to add siafund outputs: %w", err) } - if err := updateBalances(ut.tx, state.Metrics.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { - return fmt.Errorf("ApplyIndex: failed to update balances: %w", err) - } - fcDBIds, err := updateFileContractElements(ut.tx, false, state.Block, state.FileContractElements) if err != nil { return fmt.Errorf("ApplyIndex: failed to add file contracts: %w", err) } - if err := addMinerPayouts(ut.tx, state.Block.ID(), state.Block.MinerPayouts, scDBIds); err != nil { + if err := addTransactionFields(ut.tx, state.Block.Transactions, scDBIds, sfDBIds, fcDBIds, txnDBIds); err != nil { + return fmt.Errorf("ApplyIndex: failed to add transaction fields: %w", err) + } else if err := updateBalances(ut.tx, state.Metrics.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { + return fmt.Errorf("ApplyIndex: failed to update balances: %w", err) + } else if err := addMinerPayouts(ut.tx, state.Block.ID(), state.Block.MinerPayouts, scDBIds); err != nil { return fmt.Errorf("ApplyIndex: failed to add miner payouts: %w", err) - } else if txnDBIds, err := addTransactions(ut.tx, state.Block.ID(), state.Block.Transactions, scDBIds, sfDBIds, fcDBIds); err != nil { - return fmt.Errorf("ApplyIndex: failed to add transactions: addTransactions: %w", err) } else if err := updateStateTree(ut.tx, state.TreeUpdates); err != nil { return fmt.Errorf("ApplyIndex: failed to update state tree: %w", err) } else if err := addMetrics(ut.tx, state); err != nil { diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 7b909692..b458d905 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -76,8 +76,7 @@ CREATE INDEX siafund_elements_address_spent_index ON siafund_elements(address, s CREATE TABLE file_contract_elements ( id INTEGER PRIMARY KEY, block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, - -- We need to defer because transactions are inserted after file contracts - transaction_id BLOB NOT NULL REFERENCES transactions(transaction_id) ON DELETE CASCADE NOT NULL DEFERRABLE INITIALLY DEFERRED, + transaction_id BLOB REFERENCES transactions(transaction_id) ON DELETE CASCADE NOT NULL, contract_id BLOB NOT NULL, leaf_index BLOB NOT NULL, @@ -94,7 +93,7 @@ CREATE TABLE file_contract_elements ( revision_number BLOB NOT NULL, UNIQUE(contract_id, revision_number) ); -CREATE INDEX file_contract_elements_contract_id_revision_number,index ON file_contract_elements(contract_id, revision_number); +CREATE INDEX file_contract_elements_contract_id_revision_number_index ON file_contract_elements(contract_id, revision_number); CREATE TABLE last_contract_revision ( contract_id BLOB PRIMARY KEY NOT NULL, From 0a0ec7a97c5d1c930677eea4ff740893799465f7 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Wed, 9 Oct 2024 20:10:16 -0400 Subject: [PATCH 5/5] fix lint --- persist/sqlite/consensus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 4d948b5f..60831aec 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -264,8 +264,8 @@ func addTransactions(tx *txn, bid types.BlockID, txns []types.Transaction) (map[ if _, err := blockTransactionsStmt.Exec(encode(bid), txnID, i); err != nil { return nil, fmt.Errorf("failed to insert into block_transactions: %w", err) } - } + return txnDBIds, nil }