From 5edecc2bfdc23958d9455a5e2912362119233c33 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Fri, 11 Oct 2024 19:35:50 -0400 Subject: [PATCH 1/4] initial --- explorer/events.go | 13 ++- explorer/explorer.go | 2 + explorer/types.go | 18 ++++ persist/sqlite/addresses.go | 17 ++++ persist/sqlite/blocks.go | 25 ++++- persist/sqlite/consensus.go | 54 ++++++++++- persist/sqlite/init.sql | 45 ++++++++- persist/sqlite/v2consensus.go | 91 +++++++++++++++++ persist/sqlite/v2transactions.go | 161 +++++++++++++++++++++++++++++++ 9 files changed, 416 insertions(+), 10 deletions(-) create mode 100644 persist/sqlite/v2consensus.go create mode 100644 persist/sqlite/v2transactions.go diff --git a/explorer/events.go b/explorer/events.go index b51b5c21..5d3f9aae 100644 --- a/explorer/events.go +++ b/explorer/events.go @@ -11,6 +11,7 @@ import ( // event type constants const ( EventTypeTransaction = "transaction" + EventTypeV2Transaction = "v2transaction" EventTypeMinerPayout = "miner payout" EventTypeContractPayout = "contract payout" EventTypeSiafundClaim = "siafund claim" @@ -39,6 +40,9 @@ type Event struct { // EventType implements Event. func (*EventTransaction) EventType() string { return EventTypeTransaction } +// EventType implements Event. +func (*EventV2Transaction) EventType() string { return EventTypeV2Transaction } + // EventType implements Event. func (*EventMinerPayout) EventType() string { return EventTypeMinerPayout } @@ -80,6 +84,13 @@ type EventTransaction struct { Fee types.Currency `json:"fee"` } +// An EventV2Transaction represents a v2 transaction that affects the wallet. +type EventV2Transaction struct { + Transaction V2Transaction `json:"transaction"` + HostAnnouncements []chain.HostAnnouncement `json:"hostAnnouncements"` + Fee types.Currency `json:"fee"` +} + // An EventMinerPayout represents a miner payout from a block. type EventMinerPayout struct { SiacoinOutput types.SiacoinElement `json:"siacoinOutput"` @@ -206,7 +217,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate) []Event { for _, txn := range b.V2Transactions() { relevant := relevantV2Txn(txn) - var e EventTransaction + var e EventV2Transaction for _, a := range txn.Attestations { var ha chain.HostAnnouncement if ha.FromAttestation(a) { diff --git a/explorer/explorer.go b/explorer/explorer.go index fe3946b8..8852ef2d 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -52,6 +52,8 @@ type Store interface { HostMetrics() (HostMetrics, error) Transactions(ids []types.TransactionID) ([]Transaction, error) TransactionChainIndices(txid types.TransactionID, offset, limit uint64) ([]types.ChainIndex, error) + V2Transactions(ids []types.TransactionID) ([]V2Transaction, error) + V2TransactionChainIndices(txid types.TransactionID, offset, limit uint64) ([]types.ChainIndex, error) UnspentSiacoinOutputs(address types.Address, offset, limit uint64) ([]SiacoinOutput, error) UnspentSiafundOutputs(address types.Address, offset, limit uint64) ([]SiafundOutput, error) AddressEvents(address types.Address, offset, limit uint64) (events []Event, err error) diff --git a/explorer/types.go b/explorer/types.go index 820bb763..57733697 100644 --- a/explorer/types.go +++ b/explorer/types.go @@ -166,6 +166,22 @@ type Transaction struct { HostAnnouncements []chain.HostAnnouncement `json:"hostAnnouncements,omitempty"` } +// A V2Transaction is a v2 transaction that uses the wrapped types above. +type V2Transaction struct { + ID types.TransactionID `json:"id"` + ArbitraryData []byte `json:"arbitraryData,omitempty"` + + HostAnnouncements []chain.HostAnnouncement `json:"hostAnnouncements,omitempty"` +} + +// V2BlockData is a struct containing the fields from types.V2BlockData and our +// modified explorer.V2Transaction type. +type V2BlockData struct { + Height uint64 `json:"height"` + Commitment types.Hash256 `json:"commitment"` + Transactions []V2Transaction `json:"transactions"` +} + // A Block is a block containing wrapped transactions and siacoin // outputs for the miner payouts. type Block struct { @@ -175,6 +191,8 @@ type Block struct { Timestamp time.Time `json:"timestamp"` MinerPayouts []SiacoinOutput `json:"minerPayouts"` Transactions []Transaction `json:"transactions"` + + V2 *V2BlockData `json:"v2,omitempty"` } // Metrics contains various statistics relevant to the health of the Sia network. diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index a1a038d0..73166838 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -46,6 +46,23 @@ func scanEvent(tx *txn, s scanner) (ev explorer.Event, eventID int64, err error) eventTx.HostAnnouncements = append(eventTx.HostAnnouncements, announcement) } ev.Data = &eventTx + case explorer.EventTypeV2Transaction: + var txnID int64 + var eventTx explorer.EventV2Transaction + err = tx.QueryRow(`SELECT transaction_id, fee FROM v2_transaction_events WHERE event_id = ?`, eventID).Scan(&txnID, decode(&eventTx.Fee)) + if err != nil { + return explorer.Event{}, 0, fmt.Errorf("failed to fetch v2 transaction ID: %w", err) + } + txns, err := getV2Transactions(tx, map[int64]transactionID{txnID: {id: types.TransactionID(ev.ID)}}) + if err != nil || len(txns) == 0 { + return explorer.Event{}, 0, fmt.Errorf("failed to fetch v2 transaction: %w", err) + } + + rows, err := tx.Query(`SELECT public_key, net_address FROM v2_host_announcements WHERE transaction_id = ? ORDER BY transaction_order ASC`, txnID) + if err != nil { + return explorer.Event{}, 0, fmt.Errorf("failed to get host announcements: %w", err) + } + defer rows.Close() case explorer.EventTypeContractPayout: var m explorer.EventContractPayout err = tx.QueryRow(`SELECT sce.output_id, sce.leaf_index, sce.maturity_height, sce.address, sce.value, fce.contract_id, fce.leaf_index, fce.filesize, fce.file_merkle_root, fce.window_start, fce.window_end, fce.payout, fce.unlock_hash, fce.revision_number, ev.missed diff --git a/persist/sqlite/blocks.go b/persist/sqlite/blocks.go index 639a3f8a..71f2c72f 100644 --- a/persist/sqlite/blocks.go +++ b/persist/sqlite/blocks.go @@ -10,16 +10,31 @@ import ( // Block implements explorer.Store. func (s *Store) Block(id types.BlockID) (result explorer.Block, err error) { err = s.transaction(func(tx *txn) error { - err = tx.QueryRow(`SELECT parent_id, nonce, timestamp, height FROM blocks WHERE id=?`, encode(id)).Scan(decode(&result.ParentID), decode(&result.Nonce), decode(&result.Timestamp), &result.Height) - if err != nil { - return err - } - + var v2Height uint64 + var v2Commitment types.Hash256 + err = tx.QueryRow(`SELECT parent_id, nonce, timestamp, height, v2_height, v2_commitment FROM blocks WHERE id=?`, encode(id)).Scan(decode(&result.ParentID), decode(&result.Nonce), decode(&result.Timestamp), &result.Height, decodeNull(&v2Height), decodeNull(&v2Commitment)) result.MinerPayouts, err = blockMinerPayouts(tx, id) if err != nil { return fmt.Errorf("failed to get miner payouts: %w", err) } + if (v2Height != 0 && v2Commitment != types.Hash256{}) { + result.V2 = new(explorer.V2BlockData) + result.V2.Height = v2Height + result.V2.Commitment = v2Commitment + + // get block transaction IDs + transactionIDs, err := blockV2TransactionIDs(tx, id) + if err != nil { + return fmt.Errorf("failed to get block transaction IDs: %w", err) + } + + result.V2.Transactions, err = getV2Transactions(tx, transactionIDs) + if err != nil { + return fmt.Errorf("failed to get transactions: %w", err) + } + } + // get block transaction IDs transactionIDs, err := blockTransactionIDs(tx, id) if err != nil { diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 60831aec..57443275 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -18,7 +18,13 @@ type updateTx struct { func addBlock(tx *txn, b types.Block, height uint64) error { // nonce is encoded because database/sql doesn't support uint64 with high bit set - _, err := tx.Exec("INSERT INTO blocks(id, height, parent_id, nonce, timestamp) VALUES (?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encode(b.Nonce), encode(b.Timestamp)) + var v2Height uint64 + var v2Commitment types.Hash256 + if b.V2 != nil { + v2Height = b.V2.Height + v2Commitment = b.V2.Commitment + } + _, err := tx.Exec("INSERT INTO blocks(id, height, parent_id, nonce, timestamp, v2_height, v2_commitment) VALUES (?, ?, ?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encode(b.Nonce), encode(b.Timestamp), v2Height, encode(v2Commitment)) return err } @@ -595,7 +601,7 @@ func addSiafundElements(tx *txn, index types.ChainIndex, spentElements, newEleme return sfDBIds, nil } -func addEvents(tx *txn, scDBIds map[types.SiacoinOutputID]int64, fcDBIds map[explorer.DBFileContract]int64, txnDBIds map[types.TransactionID]txnDBId, events []explorer.Event) error { +func addEvents(tx *txn, scDBIds map[types.SiacoinOutputID]int64, fcDBIds map[explorer.DBFileContract]int64, txnDBIds map[types.TransactionID]txnDBId, v2TxnDBIds map[types.TransactionID]txnDBId, events []explorer.Event) error { if len(events) == 0 { return nil } @@ -624,12 +630,24 @@ func addEvents(tx *txn, scDBIds map[types.SiacoinOutputID]int64, fcDBIds map[exp } defer transactionEventStmt.Close() + v2TransactionEventStmt, err := tx.Prepare(`INSERT INTO v2_transaction_events (event_id, transaction_id, fee) VALUES (?, ?, ?)`) + if err != nil { + return fmt.Errorf("failed to prepare v2 transaction event statement: %w", err) + } + defer v2TransactionEventStmt.Close() + hostAnnouncementStmt, err := tx.Prepare(`INSERT INTO host_announcements (transaction_id, transaction_order, public_key, net_address) VALUES (?, ?, ?, ?)`) if err != nil { return fmt.Errorf("failed to prepare host anonouncement statement: %w", err) } defer hostAnnouncementStmt.Close() + v2HostAnnouncementStmt, err := tx.Prepare(`INSERT INTO v2_host_announcements (transaction_id, transaction_order, public_key, net_address) VALUES (?, ?, ?, ?)`) + if err != nil { + return fmt.Errorf("failed to prepare host anonouncement statement: %w", err) + } + defer v2HostAnnouncementStmt.Close() + minerPayoutEventStmt, err := tx.Prepare(`INSERT INTO miner_payout_events (event_id, output_id) VALUES (?, ?)`) if err != nil { return fmt.Errorf("failed to prepare miner payout event statement: %w", err) @@ -688,6 +706,29 @@ func addEvents(tx *txn, scDBIds map[types.SiacoinOutputID]int64, fcDBIds map[exp return fmt.Errorf("failed to insert host info: %w", err) } } + case *explorer.EventV2Transaction: + dbID := v2TxnDBIds[types.TransactionID(event.ID)].id + if _, err = v2TransactionEventStmt.Exec(eventID, dbID, encode(v.Fee)); err != nil { + return fmt.Errorf("failed to insert transaction event: %w", err) + } + var hosts []explorer.Host + for i, announcement := range v.HostAnnouncements { + if _, err = hostAnnouncementStmt.Exec(dbID, i, encode(announcement.PublicKey), announcement.NetAddress); err != nil { + return fmt.Errorf("failed to insert host announcement: %w", err) + } + hosts = append(hosts, explorer.Host{ + PublicKey: announcement.PublicKey, + NetAddress: announcement.NetAddress, + + KnownSince: event.Timestamp, + LastAnnouncement: event.Timestamp, + }) + } + if len(hosts) > 0 { + if err := addHosts(tx, hosts); err != nil { + return fmt.Errorf("failed to insert host info: %w", err) + } + } case *explorer.EventMinerPayout: _, err = minerPayoutEventStmt.Exec(eventID, scDBIds[types.SiacoinOutputID(event.ID)]) case *explorer.EventContractPayout: @@ -1006,6 +1047,11 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { return fmt.Errorf("ApplyIndex: failed to add transactions: %w", err) } + v2TxnDBIds, err := addV2Transactions(ut.tx, state.Block.ID(), state.Block.V2Transactions()) + if err != nil { + return fmt.Errorf("ApplyIndex: failed to add v2 transactions: %w", err) + } + scDBIds, err := addSiacoinElements( ut.tx, state.Metrics.Index, @@ -1031,6 +1077,8 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { if err := addTransactionFields(ut.tx, state.Block.Transactions, scDBIds, sfDBIds, fcDBIds, txnDBIds); err != nil { return fmt.Errorf("ApplyIndex: failed to add transaction fields: %w", err) + } else if err := addV2TransactionFields(ut.tx, state.Block.V2Transactions(), scDBIds, sfDBIds, fcDBIds, txnDBIds); err != nil { + return fmt.Errorf("ApplyIndex: failed to add v2 transaction fields: %w", err) } else if err := updateBalances(ut.tx, state.Metrics.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { return fmt.Errorf("ApplyIndex: failed to update balances: %w", err) } else if err := addMinerPayouts(ut.tx, state.Block.ID(), state.Block.MinerPayouts, scDBIds); err != nil { @@ -1039,7 +1087,7 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { return fmt.Errorf("ApplyIndex: failed to update state tree: %w", err) } else if err := addMetrics(ut.tx, state); err != nil { return fmt.Errorf("ApplyIndex: failed to update metrics: %w", err) - } else if err := addEvents(ut.tx, scDBIds, fcDBIds, txnDBIds, state.Events); err != nil { + } else if err := addEvents(ut.tx, scDBIds, fcDBIds, txnDBIds, v2TxnDBIds, state.Events); err != nil { return fmt.Errorf("ApplyIndex: failed to add events: %w", err) } else if err := updateFileContractIndices(ut.tx, false, state.Metrics.Index, state.FileContractElements); err != nil { return fmt.Errorf("ApplyIndex: failed to update file contract element indices: %w", err) diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index b458d905..a34074ca 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -8,7 +8,10 @@ CREATE TABLE blocks ( height INTEGER NOT NULL, parent_id BLOB NOT NULL, nonce BLOB NOT NULL, - timestamp INTEGER NOT NULL + timestamp INTEGER NOT NULL, + + v2_height INTEGER, + v2_commitment BLOB ); CREATE INDEX blocks_height_index ON blocks(height); @@ -250,6 +253,30 @@ CREATE TABLE transaction_file_contract_revisions ( ); CREATE INDEX transaction_file_contract_revisions_transaction_id_index ON transaction_file_contract_revisions(transaction_id); +CREATE TABLE v2_transactions ( + id INTEGER PRIMARY KEY, + transaction_id BLOB UNIQUE NOT NULL +); +CREATE INDEX v2_transactions_transaction_id_index ON v2_transactions(transaction_id); + +CREATE TABLE v2_block_transactions ( + block_id BLOB REFERENCES blocks(id) ON DELETE CASCADE NOT NULL, + transaction_id INTEGER REFERENCES v2_transactions(id) ON DELETE CASCADE NOT NULL, + block_order INTEGER NOT NULL, + UNIQUE(block_id, block_order) +); +CREATE INDEX v2_block_transactions_block_id_index ON v2_block_transactions(block_id); +CREATE INDEX v2_block_transactions_transaction_id_index ON v2_block_transactions(transaction_id); +CREATE INDEX v2_block_transactions_transaction_id_block_id ON v2_block_transactions(transaction_id, block_id); + +CREATE TABLE v2_transaction_arbitrary_data ( + transaction_id INTEGER REFERENCES v2_transactions(id) ON DELETE CASCADE NOT NULL, + data BLOB NOT NULL, + UNIQUE(transaction_id) +); + +CREATE INDEX v2_transaction_arbitrary_data_transaction_id_index ON v2_transaction_arbitrary_data(transaction_id); + CREATE TABLE state_tree ( row INTEGER NOT NULL, column INTEGER NOT NULL, @@ -309,6 +336,22 @@ CREATE TABLE foundation_subsidy_events ( output_id INTEGER REFERENCES siacoin_elements(id) ON DELETE CASCADE NOT NULL ); +CREATE TABLE v2_host_announcements ( + transaction_id INTEGER REFERENCES v2_transactions(id) ON DELETE CASCADE NOT NULL, + transaction_order INTEGER NOT NULL, + public_key BLOB NOT NULL, + net_address BLOB NOT NULL, + UNIQUE(transaction_id, transaction_order) +); +CREATE INDEX v2_host_announcements_transaction_id_index ON v2_host_announcements(transaction_id); +CREATE INDEX v2_host_announcements_public_key_index ON v2_host_announcements(public_key); + +CREATE TABLE v2_transaction_events ( + event_id INTEGER PRIMARY KEY REFERENCES events(id) ON DELETE CASCADE NOT NULL, + transaction_id INTEGER REFERENCES v2_transactions(id) ON DELETE CASCADE NOT NULL, + fee BLOB NOT NULL +); + CREATE TABLE host_info ( public_key BLOB PRIMARY KEY NOT NULL, net_address TEXT NOT NULL, diff --git a/persist/sqlite/v2consensus.go b/persist/sqlite/v2consensus.go new file mode 100644 index 00000000..43874134 --- /dev/null +++ b/persist/sqlite/v2consensus.go @@ -0,0 +1,91 @@ +package sqlite + +import ( + "database/sql" + "fmt" + + "go.sia.tech/core/types" + "go.sia.tech/explored/explorer" +) + +func addV2ArbitraryData(tx *txn, id int64, txn types.V2Transaction) error { + stmt, err := tx.Prepare(`INSERT INTO v2_transaction_arbitrary_data(transaction_id, data) VALUES (?, ?)`) + + if err != nil { + return fmt.Errorf("addV2ArbitraryData: failed to prepare statement: %w", err) + } + defer stmt.Close() + + if _, err := stmt.Exec(id, txn.ArbitraryData); err != nil { + return fmt.Errorf("addV2ArbitraryData: failed to execute statement: %w", err) + } + return nil +} + +func addV2Transactions(tx *txn, bid types.BlockID, txns []types.V2Transaction) (map[types.TransactionID]txnDBId, error) { + checkTransactionStmt, err := tx.Prepare(`SELECT id FROM v2_transactions WHERE transaction_id = ?`) + if err != nil { + return nil, fmt.Errorf("failed to prepare check v2_transaction statement: %v", err) + } + defer checkTransactionStmt.Close() + + insertTransactionStmt, err := tx.Prepare(`INSERT INTO v2_transactions (transaction_id) VALUES (?)`) + if err != nil { + return nil, fmt.Errorf("failed to prepare insert v2_transaction statement: %v", err) + } + defer insertTransactionStmt.Close() + + blockTransactionsStmt, err := tx.Prepare(`INSERT INTO v2_block_transactions(block_id, transaction_id, block_order) VALUES (?, ?, ?);`) + if err != nil { + return nil, fmt.Errorf("failed to prepare v2_block_transactions statement: %w", err) + } + defer blockTransactionsStmt.Close() + + txnDBIds := make(map[types.TransactionID]txnDBId) + for i, txn := range txns { + var exist bool + var txnID int64 + if err := checkTransactionStmt.QueryRow(encode(txn.ID())).Scan(&txnID); err != nil && err != sql.ErrNoRows { + return nil, fmt.Errorf("failed to insert v2 transaction ID: %w", err) + } else if err == nil { + exist = true + } + + if !exist { + result, err := insertTransactionStmt.Exec(encode(txn.ID())) + if err != nil { + return nil, fmt.Errorf("failed to insert into v2_transactions: %w", err) + } + txnID, err = result.LastInsertId() + if err != nil { + return nil, fmt.Errorf("failed to get v2 transaction ID: %w", err) + } + } + txnDBIds[txn.ID()] = txnDBId{id: txnID, exist: exist} + + if _, err := blockTransactionsStmt.Exec(encode(bid), txnID, i); err != nil { + return nil, fmt.Errorf("failed to insert into v2_block_transactions: %w", err) + } + } + return txnDBIds, nil +} + +func addV2TransactionFields(tx *txn, txns []types.V2Transaction, scDBIds map[types.SiacoinOutputID]int64, sfDBIds map[types.SiafundOutputID]int64, fcDBIds map[explorer.DBFileContract]int64, v2TxnDBIds map[types.TransactionID]txnDBId) error { + for _, txn := range txns { + dbID, ok := v2TxnDBIds[txn.ID()] + if !ok { + panic(fmt.Errorf("txn %v should be in txnDBIds", txn.ID())) + } + + // transaction already exists, don't reinsert its fields + if dbID.exist { + continue + } + + if err := addV2ArbitraryData(tx, dbID.id, txn); err != nil { + return fmt.Errorf("failed to add arbitrary data: %w", err) + } + } + + return nil +} diff --git a/persist/sqlite/v2transactions.go b/persist/sqlite/v2transactions.go new file mode 100644 index 00000000..666dee26 --- /dev/null +++ b/persist/sqlite/v2transactions.go @@ -0,0 +1,161 @@ +package sqlite + +import ( + "fmt" + + "go.sia.tech/core/types" + "go.sia.tech/explored/explorer" +) + +// V2TransactionChainIndices returns the chain indices of the blocks the v2 +// transaction was included in. If the transaction has not been included in +// any blocks, the result will be nil,nil. +func (s *Store) V2TransactionChainIndices(txnID types.TransactionID, offset, limit uint64) (indices []types.ChainIndex, err error) { + err = s.transaction(func(tx *txn) error { + rows, err := tx.Query(`SELECT DISTINCT b.id, b.height FROM blocks b +INNER JOIN v2_block_transactions bt ON (bt.block_id = b.id) +INNER JOIN v2_transactions t ON (t.id = bt.transaction_id) +WHERE t.transaction_id = ? +ORDER BY b.height DESC +LIMIT ? OFFSET ?`, encode(txnID), limit, offset) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var index types.ChainIndex + if err := rows.Scan(decode(&index.ID), decode(&index.Height)); err != nil { + return fmt.Errorf("failed to scan chain index: %w", err) + } + indices = append(indices, index) + } + return rows.Err() + }) + return +} + +// blockV2TransactionIDs returns the database ID for each v2 transaction in the +// block. +func blockV2TransactionIDs(tx *txn, blockID types.BlockID) (idMap map[int64]transactionID, err error) { + rows, err := tx.Query(`SELECT bt.transaction_id, block_order, t.transaction_id +FROM v2_block_transactions bt +INNER JOIN v2_transactions t ON (t.id = bt.transaction_id) +WHERE block_id = ? ORDER BY block_order ASC`, encode(blockID)) + if err != nil { + return nil, err + } + defer rows.Close() + + idMap = make(map[int64]transactionID) + for rows.Next() { + var dbID int64 + var blockOrder int64 + var txnID types.TransactionID + if err := rows.Scan(&dbID, &blockOrder, decode(&txnID)); err != nil { + return nil, fmt.Errorf("failed to scan block transaction: %w", err) + } + idMap[blockOrder] = transactionID{id: txnID, dbID: dbID} + } + return +} + +// v2TransactionArbitraryData returns the arbitrary data for each v2 transaction. +func v2TransactionArbitraryData(tx *txn, txnIDs []int64) (map[int64][]byte, error) { + query := `SELECT transaction_id, data +FROM v2_transaction_arbitrary_data +WHERE transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `)` + rows, err := tx.Query(query, queryArgs(txnIDs)...) + if err != nil { + return nil, err + } + defer rows.Close() + + result := make(map[int64][]byte) + for rows.Next() { + var txnID int64 + var data []byte + if err := rows.Scan(&txnID, &data); err != nil { + return nil, fmt.Errorf("failed to scan arbitrary data: %w", err) + } + result[txnID] = data + } + return result, nil +} + +func getV2Transactions(tx *txn, idMap map[int64]transactionID) ([]explorer.V2Transaction, error) { + dbIDs := make([]int64, len(idMap)) + for order, id := range idMap { + dbIDs[order] = id.dbID + } + + txnArbitraryData, err := v2TransactionArbitraryData(tx, dbIDs) + if err != nil { + return nil, fmt.Errorf("getV2Transactions: failed to get arbitrary data: %w", err) + } + + var results []explorer.V2Transaction + for order, dbID := range dbIDs { + txn := explorer.V2Transaction{ + ID: idMap[int64(order)].id, + ArbitraryData: txnArbitraryData[dbID], + } + + // for _, attestation := range txn.Attestations { + // var ha chain.HostAnnouncement + // if ha.FromAttestation(attestation) { + // txn.HostAnnouncements = append(txn.HostAnnouncements, ha) + // } + // } + + results = append(results, txn) + } + return results, nil +} + +// v2TransactionDatabaseIDs returns the database ID for each transaction. +func v2TransactionDatabaseIDs(tx *txn, txnIDs []types.TransactionID) (dbIDs map[int64]transactionID, err error) { + encodedIDs := func(ids []types.TransactionID) []any { + result := make([]any, len(ids)) + for i, id := range ids { + result[i] = encode(id) + } + return result + } + + query := `SELECT id, transaction_id FROM v2_transactions WHERE transaction_id IN (` + queryPlaceHolders(len(txnIDs)) + `) ORDER BY id` + rows, err := tx.Query(query, encodedIDs(txnIDs)...) + if err != nil { + return nil, err + } + defer rows.Close() + + var i int64 + dbIDs = make(map[int64]transactionID) + for rows.Next() { + var dbID int64 + var txnID types.TransactionID + if err := rows.Scan(&dbID, decode(&txnID)); err != nil { + return nil, fmt.Errorf("failed to scan transaction: %w", err) + } + dbIDs[i] = transactionID{id: txnID, dbID: dbID} + i++ + } + return +} + +// V2Transactions implements explorer.Store. +func (s *Store) V2Transactions(ids []types.TransactionID) (results []explorer.V2Transaction, err error) { + err = s.transaction(func(tx *txn) error { + dbIDs, err := v2TransactionDatabaseIDs(tx, ids) + if err != nil { + return fmt.Errorf("failed to get transaction IDs: %w", err) + } + results, err = getV2Transactions(tx, dbIDs) + if err != nil { + return fmt.Errorf("failed to get transactions: %w", err) + } + return err + }) + return +} From b6a20a34eebeaf4e5cc03c507b74df6b62bae82f Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Fri, 11 Oct 2024 19:44:41 -0400 Subject: [PATCH 2/4] add API functions --- api/client.go | 19 ++++++++++++ api/server.go | 66 ++++++++++++++++++++++++++++++++++++++-- explorer/explorer.go | 12 ++++++++ persist/sqlite/blocks.go | 5 ++- 4 files changed, 98 insertions(+), 4 deletions(-) diff --git a/api/client.go b/api/client.go index 8aac8548..ba83399b 100644 --- a/api/client.go +++ b/api/client.go @@ -130,6 +130,25 @@ func (c *Client) TransactionChainIndices(id types.TransactionID, offset, limit u return } +// V2Transaction returns the v2 transaction with the specified ID. +func (c *Client) V2Transaction(id types.TransactionID) (resp explorer.V2Transaction, err error) { + err = c.c.GET(fmt.Sprintf("/v2/transactions/%s", id), &resp) + return +} + +// V2Transactions returns the v2 transactions with the specified IDs. +func (c *Client) V2Transactions(ids []types.TransactionID) (resp []explorer.V2Transaction, err error) { + err = c.c.POST("/v2/transactions", ids, &resp) + return +} + +// V2TransactionChainIndices returns chain indices a v2 transaction was +// included in. +func (c *Client) V2TransactionChainIndices(id types.TransactionID, offset, limit uint64) (resp []types.ChainIndex, err error) { + err = c.c.GET(fmt.Sprintf("/v2/transactions/%s/indices?offset=%d&limit=%d", id, offset, limit), &resp) + return +} + // AddressSiacoinUTXOs returns the specified address' unspent outputs. func (c *Client) AddressSiacoinUTXOs(address types.Address, offset, limit uint64) (resp []explorer.SiacoinOutput, err error) { err = c.c.GET(fmt.Sprintf("/addresses/%s/utxos/siacoin?offset=%d&limit=%d", address, offset, limit), &resp) diff --git a/api/server.go b/api/server.go index 2478928d..d2a47042 100644 --- a/api/server.go +++ b/api/server.go @@ -54,6 +54,8 @@ type ( HostMetrics() (explorer.HostMetrics, error) Transactions(ids []types.TransactionID) ([]explorer.Transaction, error) TransactionChainIndices(id types.TransactionID, offset, limit uint64) ([]types.ChainIndex, error) + V2Transactions(ids []types.TransactionID) ([]explorer.V2Transaction, error) + V2TransactionChainIndices(id types.TransactionID, offset, limit uint64) ([]types.ChainIndex, error) Balance(address types.Address) (sc types.Currency, immatureSC types.Currency, sf uint64, err error) SiacoinElements(ids []types.SiacoinOutputID) (result []explorer.SiacoinOutput, err error) SiafundElements(ids []types.SiafundOutputID) (result []explorer.SiafundOutput, err error) @@ -320,6 +322,60 @@ func (s *server) transactionsBatchHandler(jc jape.Context) { jc.Encode(txns) } +func (s *server) v2TransactionsIDHandler(jc jape.Context) { + var id types.TransactionID + if jc.DecodeParam("id", &id) != nil { + return + } + txns, err := s.e.V2Transactions([]types.TransactionID{id}) + if jc.Check("failed to get transaction", err) != nil { + return + } else if len(txns) == 0 { + jc.Error(ErrTransactionNotFound, http.StatusNotFound) + return + } + jc.Encode(txns[0]) +} + +func (s *server) v2TransactionsIDIndicesHandler(jc jape.Context) { + var id types.TransactionID + if jc.DecodeParam("id", &id) != nil { + return + } + + limit := uint64(100) + offset := uint64(0) + if jc.DecodeForm("limit", &limit) != nil || jc.DecodeForm("offset", &offset) != nil { + return + } + + if limit > 500 { + limit = 500 + } + + indices, err := s.e.V2TransactionChainIndices(id, offset, limit) + if jc.Check("failed to get transaction indices", err) != nil { + return + } + jc.Encode(indices) +} + +func (s *server) v2TransactionsBatchHandler(jc jape.Context) { + var ids []types.TransactionID + if jc.Decode(&ids) != nil { + return + } else if len(ids) > maxIDs { + jc.Error(ErrTooManyIDs, http.StatusBadRequest) + return + } + + txns, err := s.e.V2Transactions(ids) + if jc.Check("failed to get transactions", err) != nil { + return + } + jc.Encode(txns) +} + func (s *server) addressessAddressUtxosSiacoinHandler(jc jape.Context) { var address types.Address if jc.DecodeParam("address", &address) != nil { @@ -554,9 +610,13 @@ func NewServer(e Explorer, cm ChainManager, s Syncer) http.Handler { "GET /blocks/:id": srv.blocksIDHandler, - "GET /transactions/:id": srv.transactionsIDHandler, - "POST /transactions": srv.transactionsBatchHandler, - "GET /transactions/:id/indices": srv.transactionsIDIndicesHandler, + "GET /transactions/:id": srv.transactionsIDHandler, + "POST /transactions": srv.transactionsBatchHandler, + "GET /transactions/:id/indices": srv.transactionsIDIndicesHandler, + + "GET /v2/transactions/:id": srv.v2TransactionsIDHandler, + "POST /v2/transactions": srv.v2TransactionsBatchHandler, + "GET /v2/transactions/:id/indices": srv.v2TransactionsIDIndicesHandler, "GET /addresses/:address/utxos/siacoin": srv.addressessAddressUtxosSiacoinHandler, "GET /addresses/:address/utxos/siafund": srv.addressessAddressUtxosSiafundHandler, diff --git a/explorer/explorer.go b/explorer/explorer.go index 8852ef2d..7eff3b80 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -213,6 +213,18 @@ func (e *Explorer) TransactionChainIndices(id types.TransactionID, offset, limit return e.s.TransactionChainIndices(id, offset, limit) } +// V2Transactions returns the v2 transactions with the specified IDs. +func (e *Explorer) V2Transactions(ids []types.TransactionID) ([]V2Transaction, error) { + return e.s.V2Transactions(ids) +} + +// V2TransactionChainIndices returns the chain indices of the blocks the +// transaction was included in. If the transaction has not been included in +// any blocks, the result will be nil,nil. +func (e *Explorer) V2TransactionChainIndices(id types.TransactionID, offset, limit uint64) ([]types.ChainIndex, error) { + return e.s.V2TransactionChainIndices(id, offset, limit) +} + // UnspentSiacoinOutputs returns the unspent siacoin outputs owned by the // specified address. func (e *Explorer) UnspentSiacoinOutputs(address types.Address, offset, limit uint64) ([]SiacoinOutput, error) { diff --git a/persist/sqlite/blocks.go b/persist/sqlite/blocks.go index 71f2c72f..028c0653 100644 --- a/persist/sqlite/blocks.go +++ b/persist/sqlite/blocks.go @@ -12,7 +12,10 @@ func (s *Store) Block(id types.BlockID) (result explorer.Block, err error) { err = s.transaction(func(tx *txn) error { var v2Height uint64 var v2Commitment types.Hash256 - err = tx.QueryRow(`SELECT parent_id, nonce, timestamp, height, v2_height, v2_commitment FROM blocks WHERE id=?`, encode(id)).Scan(decode(&result.ParentID), decode(&result.Nonce), decode(&result.Timestamp), &result.Height, decodeNull(&v2Height), decodeNull(&v2Commitment)) + err := tx.QueryRow(`SELECT parent_id, nonce, timestamp, height, v2_height, v2_commitment FROM blocks WHERE id=?`, encode(id)).Scan(decode(&result.ParentID), decode(&result.Nonce), decode(&result.Timestamp), &result.Height, decodeNull(&v2Height), decodeNull(&v2Commitment)) + if err != nil { + return fmt.Errorf("failed to get block: %w", err) + } result.MinerPayouts, err = blockMinerPayouts(tx, id) if err != nil { return fmt.Errorf("failed to get miner payouts: %w", err) From c3676ccc7bee3fb3b6bef492dd0616d82c422449 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Fri, 11 Oct 2024 20:12:25 -0400 Subject: [PATCH 3/4] fix search issue --- explorer/explorer.go | 2 +- persist/sqlite/blocks.go | 2 +- persist/sqlite/consensus.go | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/explorer/explorer.go b/explorer/explorer.go index 7eff3b80..292eaca5 100644 --- a/explorer/explorer.go +++ b/explorer/explorer.go @@ -289,7 +289,7 @@ func (e *Explorer) Search(id types.Hash256) (SearchType, error) { } _, err = e.Block(types.BlockID(id)) - if err != nil && err != sql.ErrNoRows { + if err != nil && !errors.Is(err, sql.ErrNoRows) { return SearchTypeInvalid, err } else if err == nil { return SearchTypeBlock, nil diff --git a/persist/sqlite/blocks.go b/persist/sqlite/blocks.go index 028c0653..15ece33b 100644 --- a/persist/sqlite/blocks.go +++ b/persist/sqlite/blocks.go @@ -12,7 +12,7 @@ func (s *Store) Block(id types.BlockID) (result explorer.Block, err error) { err = s.transaction(func(tx *txn) error { var v2Height uint64 var v2Commitment types.Hash256 - err := tx.QueryRow(`SELECT parent_id, nonce, timestamp, height, v2_height, v2_commitment FROM blocks WHERE id=?`, encode(id)).Scan(decode(&result.ParentID), decode(&result.Nonce), decode(&result.Timestamp), &result.Height, decodeNull(&v2Height), decodeNull(&v2Commitment)) + err := tx.QueryRow(`SELECT parent_id, nonce, timestamp, height, v2_height, v2_commitment FROM blocks WHERE id = ?`, encode(id)).Scan(decode(&result.ParentID), decode(&result.Nonce), decode(&result.Timestamp), &result.Height, decodeNull(&v2Height), decodeNull(&v2Commitment)) if err != nil { return fmt.Errorf("failed to get block: %w", err) } diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 57443275..fd4d463d 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -18,13 +18,13 @@ type updateTx struct { func addBlock(tx *txn, b types.Block, height uint64) error { // nonce is encoded because database/sql doesn't support uint64 with high bit set - var v2Height uint64 - var v2Commitment types.Hash256 + var v2Height any + var v2Commitment any if b.V2 != nil { - v2Height = b.V2.Height - v2Commitment = b.V2.Commitment + v2Height = encode(b.V2.Height) + v2Commitment = encode(b.V2.Commitment) } - _, err := tx.Exec("INSERT INTO blocks(id, height, parent_id, nonce, timestamp, v2_height, v2_commitment) VALUES (?, ?, ?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encode(b.Nonce), encode(b.Timestamp), v2Height, encode(v2Commitment)) + _, err := tx.Exec("INSERT INTO blocks(id, height, parent_id, nonce, timestamp, v2_height, v2_commitment) VALUES (?, ?, ?, ?, ?, ?, ?);", encode(b.ID()), height, encode(b.ParentID), encode(b.Nonce), encode(b.Timestamp), v2Height, v2Commitment) return err } From e06ff6a991ad12552c3c689b0fc5f3e2653a15e6 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Fri, 11 Oct 2024 20:17:19 -0400 Subject: [PATCH 4/4] add tests --- internal/testutil/chain.go | 20 ++++++ internal/testutil/check.go | 32 +++++++++ persist/sqlite/consensus.go | 2 +- persist/sqlite/v2consensus_test.go | 108 +++++++++++++++++++++++++++++ 4 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 persist/sqlite/v2consensus_test.go diff --git a/internal/testutil/chain.go b/internal/testutil/chain.go index 58f338c7..c7e04fe9 100644 --- a/internal/testutil/chain.go +++ b/internal/testutil/chain.go @@ -90,6 +90,26 @@ func MineBlock(state consensus.State, txns []types.Transaction, minerAddr types. return b } +// MineV2Block mines sets the metadata fields of the block along with the +// transactions and then generates a valid nonce for the block. +func MineV2Block(state consensus.State, txns []types.V2Transaction, minerAddr types.Address) types.Block { + b := types.Block{ + ParentID: state.Index.ID, + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: state.BlockReward()}}, + + V2: &types.V2BlockData{ + Transactions: txns, + Height: state.Index.Height + 1, + }, + } + b.V2.Commitment = state.Commitment(state.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) + for b.ID().CmpWork(state.ChildTarget) < 0 { + b.Nonce += state.NonceFactor() + } + return b +} + // SignTransactionWithContracts signs a transaction using the specified private // keys, including contract revisions. func SignTransactionWithContracts(cs consensus.State, pk, renterPK, hostPK types.PrivateKey, txn *types.Transaction) { diff --git a/internal/testutil/check.go b/internal/testutil/check.go index 95c948ff..95fbe4f3 100644 --- a/internal/testutil/check.go +++ b/internal/testutil/check.go @@ -39,6 +39,9 @@ func CheckTransaction(t *testing.T, expectTxn types.Transaction, gotTxn explorer Equal(t, "siacoin outputs", len(expectTxn.SiacoinOutputs), len(gotTxn.SiacoinOutputs)) Equal(t, "siafund inputs", len(expectTxn.SiafundInputs), len(gotTxn.SiafundInputs)) Equal(t, "siafund outputs", len(expectTxn.SiafundOutputs), len(gotTxn.SiafundOutputs)) + Equal(t, "arbitrary data", len(expectTxn.ArbitraryData), len(gotTxn.ArbitraryData)) + Equal(t, "miner fees", len(expectTxn.MinerFees), len(gotTxn.MinerFees)) + Equal(t, "signatures", len(expectTxn.Signatures), len(gotTxn.Signatures)) for i := range expectTxn.SiacoinInputs { expectSci := expectTxn.SiacoinInputs[i] @@ -100,6 +103,35 @@ func CheckTransaction(t *testing.T, expectTxn types.Transaction, gotTxn explorer } } +// CheckV2Transaction checks the inputs and outputs of the retrieved transaction +// with the source transaction. +func CheckV2Transaction(t *testing.T, expectTxn types.V2Transaction, gotTxn explorer.V2Transaction) { + t.Helper() + + Equal(t, "arbitrary data", len(expectTxn.ArbitraryData), len(gotTxn.ArbitraryData)) + + for i := range expectTxn.ArbitraryData { + Equal(t, "arbitrary data value", expectTxn.ArbitraryData[i], gotTxn.ArbitraryData[i]) + } +} + +// CheckV2ChainIndices checks that the chain indices that a v2 transaction was +// in from the explorer match the expected chain indices. +func CheckV2ChainIndices(t *testing.T, db explorer.Store, txnID types.TransactionID, expected []types.ChainIndex) { + t.Helper() + + indices, err := db.V2TransactionChainIndices(txnID, 0, 100) + switch { + case err != nil: + t.Fatal(err) + case len(indices) != len(expected): + t.Fatalf("expected %d indices, got %d", len(expected), len(indices)) + } + for i := range indices { + Equal(t, "index", expected[i], indices[i]) + } +} + // CheckFC checks the retrieved file contract with the source file contract in // addition to checking the resolved and valid fields. func CheckFC(t *testing.T, revision, resolved, valid bool, expected types.FileContract, got explorer.FileContract) { diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index fd4d463d..65fec026 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -1077,7 +1077,7 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error { if err := addTransactionFields(ut.tx, state.Block.Transactions, scDBIds, sfDBIds, fcDBIds, txnDBIds); err != nil { return fmt.Errorf("ApplyIndex: failed to add transaction fields: %w", err) - } else if err := addV2TransactionFields(ut.tx, state.Block.V2Transactions(), scDBIds, sfDBIds, fcDBIds, txnDBIds); err != nil { + } else if err := addV2TransactionFields(ut.tx, state.Block.V2Transactions(), scDBIds, sfDBIds, fcDBIds, v2TxnDBIds); err != nil { return fmt.Errorf("ApplyIndex: failed to add v2 transaction fields: %w", err) } else if err := updateBalances(ut.tx, state.Metrics.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil { return fmt.Errorf("ApplyIndex: failed to update balances: %w", err) diff --git a/persist/sqlite/v2consensus_test.go b/persist/sqlite/v2consensus_test.go new file mode 100644 index 00000000..a7688f94 --- /dev/null +++ b/persist/sqlite/v2consensus_test.go @@ -0,0 +1,108 @@ +package sqlite_test + +import ( + "path/filepath" + "testing" + + "go.sia.tech/core/types" + "go.sia.tech/coreutils" + "go.sia.tech/coreutils/chain" + ctestutil "go.sia.tech/coreutils/testutil" + "go.sia.tech/explored/internal/testutil" + "go.sia.tech/explored/persist/sqlite" + "go.uber.org/zap/zaptest" +) + +func TestV2ArbitraryData(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "explored.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := ctestutil.V2Network() + network.HardforkV2.AllowHeight = 1 + network.HardforkV2.RequireHeight = 2 + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + txn1 := types.V2Transaction{ + ArbitraryData: []byte("hello"), + } + + txn2 := types.V2Transaction{ + ArbitraryData: []byte("world"), + } + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn1, txn2}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + prev := cm.Tip() + + { + b, err := db.Block(cm.Tip().ID) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "v2 height", b.V2.Height, 1) + testutil.CheckV2Transaction(t, txn1, b.V2.Transactions[0]) + testutil.CheckV2Transaction(t, txn2, b.V2.Transactions[1]) + } + + { + dbTxns, err := db.V2Transactions([]types.TransactionID{txn1.ID(), txn2.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2Transaction(t, txn1, dbTxns[0]) + testutil.CheckV2Transaction(t, txn2, dbTxns[1]) + } + + txn3 := types.V2Transaction{ + ArbitraryData: []byte("12345"), + } + + if err := cm.AddBlocks([]types.Block{testutil.MineV2Block(cm.TipState(), []types.V2Transaction{txn1, txn2, txn3}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + { + b, err := db.Block(cm.Tip().ID) + if err != nil { + t.Fatal(err) + } + testutil.Equal(t, "v2 height", b.V2.Height, 2) + testutil.CheckV2Transaction(t, txn1, b.V2.Transactions[0]) + testutil.CheckV2Transaction(t, txn2, b.V2.Transactions[1]) + testutil.CheckV2Transaction(t, txn3, b.V2.Transactions[2]) + } + + { + dbTxns, err := db.V2Transactions([]types.TransactionID{txn1.ID(), txn2.ID(), txn3.ID()}) + if err != nil { + t.Fatal(err) + } + testutil.CheckV2Transaction(t, txn1, dbTxns[0]) + testutil.CheckV2Transaction(t, txn2, dbTxns[1]) + testutil.CheckV2Transaction(t, txn3, dbTxns[2]) + } + + testutil.CheckV2ChainIndices(t, db, txn1.ID(), []types.ChainIndex{cm.Tip(), prev}) + testutil.CheckV2ChainIndices(t, db, txn2.ID(), []types.ChainIndex{cm.Tip(), prev}) + testutil.CheckV2ChainIndices(t, db, txn3.ID(), []types.ChainIndex{cm.Tip()}) +}