diff --git a/backend/backend.go b/backend/backend.go index 0907493..b98a943 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -2,6 +2,7 @@ package backend import ( "github.com/square/beancounter/deriver" + time "time" ) // Backend is an interface which abstracts different types of backends. @@ -27,12 +28,14 @@ import ( // forgo the Finish() method and have the Accounter read from the TxResponses channel until it has // all the data it needs. This would require the Accounter to maintain its own set of transactions. type Backend interface { + ChainHeight() uint32 + AddrRequest(addr *deriver.Address) AddrResponses() <-chan *AddrResponse TxRequest(txHash string) TxResponses() <-chan *TxResponse - - ChainHeight() uint32 + BlockRequest(height uint32) + BlockResponses() <-chan *BlockResponse Finish() } @@ -51,6 +54,11 @@ type TxResponse struct { Hex string } +type BlockResponse struct { + Height uint32 + Timestamp time.Time +} + // HasTransactions returns true if the Response contains any transactions func (r *AddrResponse) HasTransactions() bool { return len(r.TxHashes) > 0 diff --git a/backend/btcd_backend.go b/backend/btcd_backend.go index 15e62af..988a7fc 100644 --- a/backend/btcd_backend.go +++ b/backend/btcd_backend.go @@ -17,6 +17,8 @@ import ( // balance and transaction history information for a given address. // BtcdBackend implements Backend interface. type BtcdBackend struct { + chainHeight uint32 + client *rpcclient.Client network utils.Network blockHeightMu sync.Mutex // mutex to guard read/writes to blockHeightLookup map @@ -28,7 +30,10 @@ type BtcdBackend struct { txRequests chan string txResponses chan *TxResponse - chainHeight uint32 + // channels used to communicate with the Blockfinder + blockRequests chan uint32 + blockResponses chan *BlockResponse + // internal channels transactionsMu sync.Mutex // mutex to guard read/writes to transactions map cachedTransactions map[string]*TxResponse @@ -42,6 +47,7 @@ const ( maxTxsPerAddr = 1000 addrRequestsChanSize = 100 + blockRequestChanSize = 100 concurrency = 100 ) @@ -72,7 +78,6 @@ func NewBtcdBackend(hostPort, user, pass string, network utils.Network) (*BtcdBa if genesis.String() != utils.GenesisBlock(network) { return nil, errors.New(fmt.Sprintf("Unexpected genesis block %s != %s", genesis.String(), utils.GenesisBlock(network))) } - fmt.Printf("%+v\n", genesis) height, err := client.GetBlockCount() if err != nil { @@ -80,13 +85,16 @@ func NewBtcdBackend(hostPort, user, pass string, network utils.Network) (*BtcdBa } b := &BtcdBackend{ - client: client, - network: network, - chainHeight: uint32(height), - addrRequests: make(chan *deriver.Address, addrRequestsChanSize), - addrResponses: make(chan *AddrResponse, addrRequestsChanSize), - txRequests: make(chan string, 2*maxTxsPerAddr), - txResponses: make(chan *TxResponse, 2*maxTxsPerAddr), + client: client, + network: network, + chainHeight: uint32(height), + addrRequests: make(chan *deriver.Address, addrRequestsChanSize), + addrResponses: make(chan *AddrResponse, addrRequestsChanSize), + txRequests: make(chan string, 2*maxTxsPerAddr), + txResponses: make(chan *TxResponse, 2*maxTxsPerAddr), + blockRequests: make(chan uint32, 2*blockRequestChanSize), + blockResponses: make(chan *BlockResponse, 2*blockRequestChanSize), + blockHeightLookup: make(map[string]int64), cachedTransactions: make(map[string]*TxResponse), doneCh: make(chan bool), @@ -129,6 +137,14 @@ func (b *BtcdBackend) TxResponses() <-chan *TxResponse { return b.txResponses } +func (b *BtcdBackend) BlockRequest(height uint32) { + b.blockRequests <- height +} + +func (b *BtcdBackend) BlockResponses() <-chan *BlockResponse { + return b.blockResponses +} + // Finish informs the backend to stop doing its work. func (b *BtcdBackend) Finish() { close(b.doneCh) @@ -152,6 +168,11 @@ func (b *BtcdBackend) processRequests() { if err != nil { panic(fmt.Sprintf("processTxRequest failed: %+v", err)) } + case block := <-b.blockRequests: + err := b.processBlockRequest(block) + if err != nil { + panic(fmt.Sprintf("processBlockRequest failed: %+v", err)) + } case <-b.doneCh: break } @@ -234,6 +255,36 @@ func (b *BtcdBackend) processTxRequest(txHash string) error { return nil } +func (b *BtcdBackend) processBlockRequest(height uint32) error { + hash, err := b.client.GetBlockHash(int64(height)) + if err != nil { + if jerr, ok := err.(*btcjson.RPCError); ok { + switch jerr.Code { + case btcjson.ErrRPCInvalidAddressOrKey: + return errors.Wrap(err, fmt.Sprintf("blockchain doesn't have block %d", height)) + } + } + return errors.Wrap(err, fmt.Sprintf("could not fetch block %d", height)) + } + + header, err := b.client.GetBlockHeader(hash) + if err != nil { + if jerr, ok := err.(*btcjson.RPCError); ok { + switch jerr.Code { + case btcjson.ErrRPCInvalidAddressOrKey: + return errors.Wrap(err, fmt.Sprintf("blockchain doesn't have block %d", height)) + } + } + return errors.Wrap(err, fmt.Sprintf("could not fetch block %d", height)) + } + + b.blockResponses <- &BlockResponse{ + Height: height, + Timestamp: header.Timestamp, + } + return nil +} + func (b *BtcdBackend) cacheTxs(txs []*btcjson.SearchRawTransactionsResult) { for _, tx := range txs { b.transactionsMu.Lock() diff --git a/backend/common.go b/backend/common.go index d0bac3d..5e639b0 100644 --- a/backend/common.go +++ b/backend/common.go @@ -2,6 +2,7 @@ package backend import ( "github.com/square/beancounter/utils" + "time" ) // index, address and transaction and helper structs used by recorder and fixture @@ -11,6 +12,7 @@ type index struct { Metadata metadata `json:"metadata"` Addresses []address `json:"addresses"` Transactions []transaction `json:"transactions"` + Blocks []block `json:"blocks"` } type metadata struct { @@ -43,3 +45,8 @@ type byTransactionID []transaction func (a byTransactionID) Len() int { return len(a) } func (a byTransactionID) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a byTransactionID) Less(i, j int) bool { return a[i].Hash < a[j].Hash } + +type block struct { + Height uint32 `json:"height"` + Timestamp time.Time `json:"timestamp"` +} diff --git a/backend/electrum/blockchain.go b/backend/electrum/blockchain.go index 5565431..6795adc 100644 --- a/backend/electrum/blockchain.go +++ b/backend/electrum/blockchain.go @@ -118,6 +118,12 @@ type Header struct { Hex string `json:"hex"` } +type Block struct { + Count uint `json:"count"` + Hex string `json:"hex"` + Max uint `json:"max"` +} + func NewNode(addr, port string, network utils.Network) (*Node, error) { n := &Node{} var a string @@ -261,6 +267,13 @@ func (n *Node) ServerPeersSubscribe() ([]Peer, error) { return out, nil } +// BlockchainBlockHeaders returns a block header (160 hex). +func (n *Node) BlockchainBlockHeaders(height uint32, count uint) (Block, error) { + var block Block + err := n.request("blockchain.block.headers", []interface{}{height, count}, &block) + return block, err +} + func (n *Node) request(method string, params []interface{}, result interface{}) error { msg := RequestMessage{ Id: atomic.AddUint64(&n.nextId, 1), diff --git a/backend/electrum_backend.go b/backend/electrum_backend.go index 31373a0..d76c8f7 100644 --- a/backend/electrum_backend.go +++ b/backend/electrum_backend.go @@ -1,8 +1,11 @@ package backend import ( + "bytes" + "encoding/hex" "errors" "fmt" + "github.com/btcsuite/btcd/wire" "log" "strconv" "strings" @@ -24,12 +27,14 @@ import ( // - has crossed the height we are interested in. // - we then negotiate protocol v1.2 // -// A background goroutine continously connects to peers. +// A background goroutine continuously connects to peers. // ElectrumBackend wraps Electrum node and its API to provide a simple // balance and transaction history information for a given address. // ElectrumBackend implements Backend interface. type ElectrumBackend struct { + chainHeight uint32 + // peer management nodeMu sync.RWMutex // mutex to guard reads/writes to nodes map nodes map[string]*electrum.Node @@ -44,13 +49,15 @@ type ElectrumBackend struct { txResponses chan *TxResponse txRequests chan string + // channels used to communicate with the Blockfinder + blockRequests chan uint32 + blockResponses chan *BlockResponse + // internal channels peersRequests chan struct{} transactionsMu sync.Mutex // mutex to guard read/writes to transactions map transactions map[string]int64 doneCh chan bool - - chainHeight uint32 } const ( @@ -83,6 +90,8 @@ func NewElectrumBackend(addr, port string, network utils.Network) (*ElectrumBack addrResponses: make(chan *AddrResponse, 2*maxPeers), txRequests: make(chan string, 2*maxPeers), txResponses: make(chan *TxResponse, 2*maxPeers), + blockRequests: make(chan uint32, 2*maxPeers), + blockResponses: make(chan *BlockResponse, 2*maxPeers), peersRequests: make(chan struct{}), transactions: make(map[string]int64), @@ -148,6 +157,14 @@ func (eb *ElectrumBackend) TxResponses() <-chan *TxResponse { return eb.txResponses } +func (eb *ElectrumBackend) BlockRequest(height uint32) { + eb.blockRequests <- height +} + +func (eb *ElectrumBackend) BlockResponses() <-chan *BlockResponse { + return eb.blockResponses +} + // Finish informs the backend to stop doing its work. func (eb *ElectrumBackend) Finish() { close(eb.doneCh) @@ -292,6 +309,11 @@ func (eb *ElectrumBackend) processRequests(node *electrum.Node) { if err != nil { return } + case block := <-eb.blockRequests: + err := eb.processBlockRequest(node, block) + if err != nil { + return + } } } } @@ -356,6 +378,42 @@ func (eb *ElectrumBackend) getTxHeight(txHash string) (int64, error) { return height, nil } +// note: we could be more efficient and batch things up. +func (eb *ElectrumBackend) processBlockRequest(node *electrum.Node, height uint32) error { + block, err := node.BlockchainBlockHeaders(height, 1) + if err != nil { + log.Printf("processBlockRequest failed with: %s, %+v", node.Ident, err) + eb.removeNode(node.Ident) + + // requeue request + // TODO: we should have a retry counter and fail gracefully if an address fails too + // many times. + eb.blockRequests <- height + return err + } + + // Decode hex to get Timestamp + b, err := hex.DecodeString(block.Hex) + if err != nil { + fmt.Printf("failed to unhex block %d: %s\n", height, block.Hex) + panic(err) + } + + var blockHeader wire.BlockHeader + err = blockHeader.Deserialize(bytes.NewReader(b)) + if err != nil { + fmt.Printf("failed to parse block %d: %s\n", height, block.Hex) + panic(err) + } + + eb.blockResponses <- &BlockResponse{ + Height: height, + Timestamp: blockHeader.Timestamp, + } + + return nil +} + func (eb *ElectrumBackend) processAddrRequest(node *electrum.Node, addr *deriver.Address) error { txs, err := node.BlockchainAddressGetHistory(addr.String()) if err != nil { diff --git a/backend/fixture_backend.go b/backend/fixture_backend.go index aab5a44..9769940 100644 --- a/backend/fixture_backend.go +++ b/backend/fixture_backend.go @@ -2,6 +2,7 @@ package backend import ( "encoding/json" + "fmt" "io/ioutil" "os" "sync" @@ -14,10 +15,12 @@ import ( // FixtureBackend loads data from a file that was previously recorded by // RecorderBackend type FixtureBackend struct { - addrIndexMu sync.Mutex - addrIndex map[string]AddrResponse - txIndexMu sync.Mutex - txIndex map[string]TxResponse + addrIndexMu sync.Mutex + addrIndex map[string]AddrResponse + txIndexMu sync.Mutex + txIndex map[string]TxResponse + blockIndexMu sync.Mutex + blockIndex map[uint32]BlockResponse // channels used to communicate with the Accounter addrRequests chan *deriver.Address @@ -25,6 +28,10 @@ type FixtureBackend struct { txRequests chan string txResponses chan *TxResponse + // channels used to communicate with the Blockfinder + blockRequests chan uint32 + blockResponses chan *BlockResponse + transactionsMu sync.Mutex // mutex to guard read/writes to transactions map transactions map[string]int64 @@ -39,14 +46,17 @@ type FixtureBackend struct { // NewFixtureBackend returns a new FixtureBackend structs or errors. func NewFixtureBackend(filepath string) (*FixtureBackend, error) { cb := &FixtureBackend{ - addrRequests: make(chan *deriver.Address, 10), - addrResponses: make(chan *AddrResponse, 10), - txRequests: make(chan string, 1000), - txResponses: make(chan *TxResponse, 1000), - addrIndex: make(map[string]AddrResponse), - txIndex: make(map[string]TxResponse), - transactions: make(map[string]int64), - doneCh: make(chan bool), + addrRequests: make(chan *deriver.Address, 10), + addrResponses: make(chan *AddrResponse, 10), + txRequests: make(chan string, 1000), + txResponses: make(chan *TxResponse, 1000), + blockRequests: make(chan uint32, 10), + blockResponses: make(chan *BlockResponse, 10), + addrIndex: make(map[string]AddrResponse), + txIndex: make(map[string]TxResponse), + blockIndex: make(map[uint32]BlockResponse), + transactions: make(map[string]int64), + doneCh: make(chan bool), } f, err := os.Open(filepath) @@ -79,6 +89,10 @@ func (b *FixtureBackend) TxRequest(txHash string) { b.txRequests <- txHash } +func (b *FixtureBackend) BlockRequest(height uint32) { + b.blockRequests <- height +} + // AddrResponses exposes a channel that allows to consume backend's responses to // address requests created with AddrRequest() func (b *FixtureBackend) AddrResponses() <-chan *AddrResponse { @@ -93,6 +107,10 @@ func (b *FixtureBackend) TxResponses() <-chan *TxResponse { return b.txResponses } +func (b *FixtureBackend) BlockResponses() <-chan *BlockResponse { + return b.blockResponses +} + // Finish informs the backend to stop doing its work. func (b *FixtureBackend) Finish() { close(b.doneCh) @@ -121,6 +139,8 @@ func (b *FixtureBackend) processRequests() { continue } b.txResponses <- txResp + case block := <-b.blockRequests: + b.processBlockRequest(block) case <-b.doneCh: return } @@ -156,7 +176,19 @@ func (b *FixtureBackend) processTxRequest(txHash string) { // assuming that transaction does not exist in the fixture file } -func (b *FixtureBackend) loadFromFile(f *os.File) error { +func (b *FixtureBackend) processBlockRequest(height uint32) { + b.blockIndexMu.Lock() + resp, exists := b.blockIndex[height] + b.blockIndexMu.Unlock() + + if exists { + b.blockResponses <- &resp + return + } + panic(fmt.Sprintf("fixture doesn't contain block %d", height)) +} + +func (fb *FixtureBackend) loadFromFile(f *os.File) error { var cachedData index byteValue, err := ioutil.ReadAll(f) @@ -169,24 +201,31 @@ func (b *FixtureBackend) loadFromFile(f *os.File) error { return err } - b.height = cachedData.Metadata.Height + fb.height = cachedData.Metadata.Height for _, addr := range cachedData.Addresses { a := AddrResponse{ Address: deriver.NewAddress(addr.Path, addr.Address, addr.Network, addr.Change, addr.AddressIndex), TxHashes: addr.TxHashes, } - b.addrIndex[addr.Address] = a + fb.addrIndex[addr.Address] = a } for _, tx := range cachedData.Transactions { - b.txIndex[tx.Hash] = TxResponse{ + fb.txIndex[tx.Hash] = TxResponse{ Hash: tx.Hash, Height: tx.Height, Hex: tx.Hex, } - b.transactions[tx.Hash] = tx.Height + fb.transactions[tx.Hash] = tx.Height + } + + for _, b := range cachedData.Blocks { + fb.blockIndex[b.Height] = BlockResponse{ + Height: b.Height, + Timestamp: b.Timestamp, + } } return nil diff --git a/backend/recorder_backend.go b/backend/recorder_backend.go index 391909e..9e78bcc 100644 --- a/backend/recorder_backend.go +++ b/backend/recorder_backend.go @@ -15,16 +15,21 @@ import ( // balance and transaction history information for a given address. // RecorderBackend implements Backend interface. type RecorderBackend struct { - backend Backend - addrIndexMu sync.Mutex - addrIndex map[string]AddrResponse - txIndexMu sync.Mutex - txIndex map[string]TxResponse + backend Backend + addrIndexMu sync.Mutex + addrIndex map[string]AddrResponse + txIndexMu sync.Mutex + txIndex map[string]TxResponse + blockIndexMu sync.Mutex + blockIndex map[uint32]BlockResponse // channels used to communicate with the Accounter addrResponses chan *AddrResponse txResponses chan *TxResponse + // channels used to communicate with the Blockfinder + blockResponses chan *BlockResponse + // internal channels doneCh chan bool @@ -40,8 +45,10 @@ func NewRecorderBackend(b Backend, filepath string) (*RecorderBackend, error) { backend: b, addrResponses: make(chan *AddrResponse, addrRequestsChanSize), txResponses: make(chan *TxResponse, 2*maxTxsPerAddr), + blockResponses: make(chan *BlockResponse, blockRequestChanSize), addrIndex: make(map[string]AddrResponse), txIndex: make(map[string]TxResponse), + blockIndex: make(map[uint32]BlockResponse), doneCh: make(chan bool), outputFilepath: filepath, } @@ -76,6 +83,14 @@ func (rb *RecorderBackend) TxResponses() <-chan *TxResponse { return rb.txResponses } +func (rb *RecorderBackend) BlockRequest(height uint32) { + rb.backend.BlockRequest(height) +} + +func (rb *RecorderBackend) BlockResponses() <-chan *BlockResponse { + return rb.blockResponses +} + // Finish informs the backend to stop doing its work. func (rb *RecorderBackend) Finish() { rb.backend.Finish() @@ -93,6 +108,7 @@ func (rb *RecorderBackend) ChainHeight() uint32 { func (rb *RecorderBackend) processRequests() { backendAddrResponses := rb.backend.AddrResponses() backendTxResponses := rb.backend.TxResponses() + backendBlockResponses := rb.backend.BlockResponses() for { select { @@ -114,6 +130,15 @@ func (rb *RecorderBackend) processRequests() { rb.txIndex[txResp.Hash] = *txResp rb.txIndexMu.Unlock() rb.txResponses <- txResp + case block, ok := <-backendBlockResponses: + if !ok { + backendBlockResponses = nil + continue + } + rb.blockIndexMu.Lock() + rb.blockIndex[block.Height] = *block + rb.blockIndexMu.Unlock() + rb.blockResponses <- block case <-rb.doneCh: return } @@ -121,7 +146,10 @@ func (rb *RecorderBackend) processRequests() { } func (rb *RecorderBackend) writeToFile() error { - cachedData := index{Metadata: metadata{}, Addresses: []address{}, Transactions: []transaction{}} + cachedData := index{ + Metadata: metadata{}, Addresses: []address{}, Transactions: []transaction{}, + Blocks: []block{}, + } reporter.GetInstance().Logf("writing data to %s\n ...", rb.outputFilepath) f, err := os.Create(rb.outputFilepath) @@ -156,6 +184,13 @@ func (rb *RecorderBackend) writeToFile() error { } sort.Sort(byTransactionID(cachedData.Transactions)) + for _, b := range rb.blockIndex { + cachedData.Blocks = append(cachedData.Blocks, block{ + Height: b.Height, + Timestamp: b.Timestamp, + }) + } + cachedDataJSON, err := json.MarshalIndent(cachedData, "", " ") if err != nil { return err diff --git a/blockfinder/blockfinder.go b/blockfinder/blockfinder.go new file mode 100644 index 0000000..1e219a8 --- /dev/null +++ b/blockfinder/blockfinder.go @@ -0,0 +1,91 @@ +package blockfinder + +import ( + "fmt" + "github.com/square/beancounter/backend" + "sort" + "time" +) + +// Blockfinder uses the backend to find the last block before a given timestamp. +// For each block, the block's time is computed by taking the median of the previous 11 blocks. +type Blockfinder struct { + blocks map[uint32]time.Time + backend backend.Backend + blockResponses <-chan *backend.BlockResponse +} + +// New instantiates a new Blockfinder +func New(b backend.Backend) *Blockfinder { + bf := &Blockfinder{ + backend: b, + } + bf.blocks = make(map[uint32]time.Time) + bf.blockResponses = b.BlockResponses() + return bf +} + +// Returns block height, block median, block timestamp +func (bf *Blockfinder) Search(timestamp time.Time) (uint32, time.Time, time.Time) { + target := timestamp.Unix() + + min := uint32(10) // any small number above 5 works + minMedian := bf.searchSync(min) + + // Use chainheight - 6 (because of min confirmations) - 5 (because of the way we compute median) + max := bf.backend.ChainHeight() - 11 + maxMedian := bf.searchSync(max) + + for max-min > 1 { + avg := (max + min) / 2 + avgTimestamp := bf.searchSync(avg) + fmt.Printf("min: %d %d, avg: %d %d, max: %d %d, target: %d\n", + min, minMedian, avg, avgTimestamp, max, maxMedian, target) + + if avgTimestamp < minMedian || avgTimestamp > maxMedian { + panic("non-monotonic medians") + } + + if target == avgTimestamp { + min = avg + minMedian = avgTimestamp + break + } else if target > avgTimestamp { + min = avg + minMedian = avgTimestamp + } else { + max = avg + maxMedian = avgTimestamp + } + } + + bf.backend.BlockRequest(min) + blockHeader := <-bf.blockResponses + + // Give recorder backend a chance to write the data + bf.backend.Finish() + + return min, time.Unix(minMedian, 0), blockHeader.Timestamp +} + +// TODO: cache requests +// around 283655 is a good test case for this function... +// We define the median time as the median of time timestamps from 5 blocks before and 5 blocks +// after. We have to pick a total of 11 blocks, because that's how the validation rule is defined. +// (https://en.bitcoin.it/wiki/Block_timestamp, https://github.com/bitcoin/bitcoin/blob/0.17/src/chain.h#L307) +// but we don't have to do the previous 11. Any consecutive 11 blocks has monotonic medians. By looking +// at the previous 5 and next 5, we reduce the delta between the block time displayed on a website +// such as live.blockcypher.com and the median we compute. It makes things less confusing for people +// who might not understand why we need to look at the median. +func (bf *Blockfinder) searchSync(height uint32) int64 { + for i := height - 5; i <= (height + 5); i++ { + bf.backend.BlockRequest(i) + } + timestamps := []int64{} + for i := 0; i < 11; i++ { + blockHeader := <-bf.blockResponses + timestamps = append(timestamps, blockHeader.Timestamp.Unix()) + } + sort.Slice(timestamps, func(i, j int) bool { return timestamps[i] < timestamps[j] }) + return timestamps[5] +} diff --git a/blockfinder/blockfinder_test.go b/blockfinder/blockfinder_test.go new file mode 100644 index 0000000..e730b84 --- /dev/null +++ b/blockfinder/blockfinder_test.go @@ -0,0 +1,20 @@ +package blockfinder + +import ( + "github.com/square/beancounter/backend" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestFindBLock(t *testing.T) { + b, err := backend.NewFixtureBackend("../fixtures/blocks.json") + assert.NoError(t, err) + + bf := New(b) + height, median, timestamp := bf.Search(time.Unix(1533153600, 0)) + + assert.Equal(t, height, uint32(534733)) + assert.Equal(t, median.Unix(), int64(1533152846)) + assert.Equal(t, timestamp.Unix(), int64(1533152846)) +} diff --git a/fixtures/blocks.json b/fixtures/blocks.json new file mode 100644 index 0000000..17c2dcd --- /dev/null +++ b/fixtures/blocks.json @@ -0,0 +1,781 @@ +{ + "metadata": { + "height": 546110 + }, + "addresses": [], + "transactions": [], + "blocks": [ + { + "height": 533300, + "timestamp": "2018-07-23T09:00:55-07:00" + }, + { + "height": 533296, + "timestamp": "2018-07-23T08:42:44-07:00" + }, + { + "height": 534731, + "timestamp": "2018-08-01T12:35:25-07:00" + }, + { + "height": 534728, + "timestamp": "2018-08-01T11:59:43-07:00" + }, + { + "height": 546094, + "timestamp": "2018-10-16T20:21:33-07:00" + }, + { + "height": 533295, + "timestamp": "2018-07-23T08:42:08-07:00" + }, + { + "height": 535435, + "timestamp": "2018-08-05T19:32:32-07:00" + }, + { + "height": 534700, + "timestamp": "2018-08-01T07:03:39-07:00" + }, + { + "height": 534725, + "timestamp": "2018-08-01T11:22:00-07:00" + }, + { + "height": 477837, + "timestamp": "2017-07-27T11:31:50-07:00" + }, + { + "height": 537568, + "timestamp": "2018-08-19T16:58:47-07:00" + }, + { + "height": 537570, + "timestamp": "2018-08-19T17:06:00-07:00" + }, + { + "height": 533294, + "timestamp": "2018-07-23T08:27:11-07:00" + }, + { + "height": 15, + "timestamp": "2009-01-09T20:45:46-08:00" + }, + { + "height": 409575, + "timestamp": "2016-04-30T06:44:47-07:00" + }, + { + "height": 477839, + "timestamp": "2017-07-27T11:34:11-07:00" + }, + { + "height": 511966, + "timestamp": "2018-03-04T05:44:23-08:00" + }, + { + "height": 529029, + "timestamp": "2018-06-24T06:48:56-07:00" + }, + { + "height": 534367, + "timestamp": "2018-07-30T01:02:11-07:00" + }, + { + "height": 534763, + "timestamp": "2018-08-01T17:07:00-07:00" + }, + { + "height": 534635, + "timestamp": "2018-07-31T20:13:52-07:00" + }, + { + "height": 477834, + "timestamp": "2017-07-27T11:04:27-07:00" + }, + { + "height": 511973, + "timestamp": "2018-03-04T06:26:10-08:00" + }, + { + "height": 511965, + "timestamp": "2018-03-04T05:41:02-08:00" + }, + { + "height": 533301, + "timestamp": "2018-07-23T09:07:43-07:00" + }, + { + "height": 533304, + "timestamp": "2018-07-23T10:14:47-07:00" + }, + { + "height": 535436, + "timestamp": "2018-08-05T20:18:18-07:00" + }, + { + "height": 534630, + "timestamp": "2018-07-31T19:05:01-07:00" + }, + { + "height": 5, + "timestamp": "2009-01-08T19:23:48-08:00" + }, + { + "height": 273054, + "timestamp": "2013-12-04T09:16:15-08:00" + }, + { + "height": 529035, + "timestamp": "2018-06-24T07:55:41-07:00" + }, + { + "height": 537563, + "timestamp": "2018-08-19T16:00:28-07:00" + }, + { + "height": 534745, + "timestamp": "2018-08-01T15:24:37-07:00" + }, + { + "height": 534736, + "timestamp": "2018-08-01T13:20:08-07:00" + }, + { + "height": 477838, + "timestamp": "2017-07-27T11:32:53-07:00" + }, + { + "height": 511971, + "timestamp": "2018-03-04T06:19:56-08:00" + }, + { + "height": 535428, + "timestamp": "2018-08-05T18:55:22-07:00" + }, + { + "height": 534896, + "timestamp": "2018-08-02T13:07:07-07:00" + }, + { + "height": 534694, + "timestamp": "2018-08-01T05:59:06-07:00" + }, + { + "height": 534730, + "timestamp": "2018-08-01T12:25:35-07:00" + }, + { + "height": 534767, + "timestamp": "2018-08-01T17:27:13-07:00" + }, + { + "height": 546096, + "timestamp": "2018-10-16T20:51:12-07:00" + }, + { + "height": 546099, + "timestamp": "2018-10-16T21:30:25-07:00" + }, + { + "height": 409581, + "timestamp": "2016-04-30T07:30:26-07:00" + }, + { + "height": 529031, + "timestamp": "2018-06-24T07:17:08-07:00" + }, + { + "height": 534370, + "timestamp": "2018-07-30T01:14:58-07:00" + }, + { + "height": 534631, + "timestamp": "2018-07-31T19:18:04-07:00" + }, + { + "height": 534632, + "timestamp": "2018-07-31T19:36:14-07:00" + }, + { + "height": 534699, + "timestamp": "2018-08-01T06:52:11-07:00" + }, + { + "height": 534749, + "timestamp": "2018-08-01T15:44:22-07:00" + }, + { + "height": 6, + "timestamp": "2009-01-08T19:29:49-08:00" + }, + { + "height": 409580, + "timestamp": "2016-04-30T07:13:46-07:00" + }, + { + "height": 529033, + "timestamp": "2018-06-24T07:35:37-07:00" + }, + { + "height": 534366, + "timestamp": "2018-07-30T00:12:54-07:00" + }, + { + "height": 534744, + "timestamp": "2018-08-01T15:20:00-07:00" + }, + { + "height": 546095, + "timestamp": "2018-10-16T20:31:45-07:00" + }, + { + "height": 409574, + "timestamp": "2016-04-30T06:43:18-07:00" + }, + { + "height": 273050, + "timestamp": "2013-12-04T08:27:28-08:00" + }, + { + "height": 511964, + "timestamp": "2018-03-04T05:36:00-08:00" + }, + { + "height": 533298, + "timestamp": "2018-07-23T08:49:32-07:00" + }, + { + "height": 534368, + "timestamp": "2018-07-30T01:02:53-07:00" + }, + { + "height": 534369, + "timestamp": "2018-07-30T01:07:40-07:00" + }, + { + "height": 534693, + "timestamp": "2018-08-01T05:47:26-07:00" + }, + { + "height": 534743, + "timestamp": "2018-08-01T15:04:47-07:00" + }, + { + "height": 534894, + "timestamp": "2018-08-02T12:51:43-07:00" + }, + { + "height": 10, + "timestamp": "2009-01-08T20:05:52-08:00" + }, + { + "height": 546104, + "timestamp": "2018-10-16T22:30:25-07:00" + }, + { + "height": 273058, + "timestamp": "2013-12-04T09:55:46-08:00" + }, + { + "height": 511970, + "timestamp": "2018-03-04T06:08:35-08:00" + }, + { + "height": 529030, + "timestamp": "2018-06-24T07:16:42-07:00" + }, + { + "height": 535429, + "timestamp": "2018-08-05T19:00:45-07:00" + }, + { + "height": 534365, + "timestamp": "2018-07-30T00:09:57-07:00" + }, + { + "height": 534636, + "timestamp": "2018-07-31T20:19:27-07:00" + }, + { + "height": 537569, + "timestamp": "2018-08-19T17:04:08-07:00" + }, + { + "height": 535431, + "timestamp": "2018-08-05T19:06:24-07:00" + }, + { + "height": 9, + "timestamp": "2009-01-08T19:54:39-08:00" + }, + { + "height": 546098, + "timestamp": "2018-10-16T21:10:28-07:00" + }, + { + "height": 546100, + "timestamp": "2018-10-16T21:31:23-07:00" + }, + { + "height": 273051, + "timestamp": "2013-12-04T08:24:52-08:00" + }, + { + "height": 535432, + "timestamp": "2018-08-05T19:12:26-07:00" + }, + { + "height": 14, + "timestamp": "2009-01-08T20:33:09-08:00" + }, + { + "height": 546103, + "timestamp": "2018-10-16T22:04:23-07:00" + }, + { + "height": 534363, + "timestamp": "2018-07-29T23:52:13-07:00" + }, + { + "height": 534903, + "timestamp": "2018-08-02T14:48:04-07:00" + }, + { + "height": 534762, + "timestamp": "2018-08-01T16:50:36-07:00" + }, + { + "height": 534761, + "timestamp": "2018-08-01T16:47:23-07:00" + }, + { + "height": 529037, + "timestamp": "2018-06-24T08:02:00-07:00" + }, + { + "height": 534627, + "timestamp": "2018-07-31T18:39:49-07:00" + }, + { + "height": 534741, + "timestamp": "2018-08-01T14:32:19-07:00" + }, + { + "height": 534752, + "timestamp": "2018-08-01T16:09:06-07:00" + }, + { + "height": 12, + "timestamp": "2009-01-08T20:21:28-08:00" + }, + { + "height": 409577, + "timestamp": "2016-04-30T06:52:32-07:00" + }, + { + "height": 529034, + "timestamp": "2018-06-24T07:37:17-07:00" + }, + { + "height": 535430, + "timestamp": "2018-08-05T19:05:15-07:00" + }, + { + "height": 534897, + "timestamp": "2018-08-02T13:10:34-07:00" + }, + { + "height": 534902, + "timestamp": "2018-08-02T14:31:21-07:00" + }, + { + "height": 534629, + "timestamp": "2018-07-31T19:04:21-07:00" + }, + { + "height": 534361, + "timestamp": "2018-07-29T23:30:51-07:00" + }, + { + "height": 409571, + "timestamp": "2016-04-30T06:35:06-07:00" + }, + { + "height": 409576, + "timestamp": "2016-04-30T06:48:53-07:00" + }, + { + "height": 477835, + "timestamp": "2017-07-27T11:10:14-07:00" + }, + { + "height": 529032, + "timestamp": "2018-06-24T07:35:10-07:00" + }, + { + "height": 529036, + "timestamp": "2018-06-24T08:00:07-07:00" + }, + { + "height": 535427, + "timestamp": "2018-08-05T18:44:40-07:00" + }, + { + "height": 534362, + "timestamp": "2018-07-29T23:47:30-07:00" + }, + { + "height": 534628, + "timestamp": "2018-07-31T18:52:59-07:00" + }, + { + "height": 534733, + "timestamp": "2018-08-01T12:47:26-07:00" + }, + { + "height": 273053, + "timestamp": "2013-12-04T09:01:05-08:00" + }, + { + "height": 273055, + "timestamp": "2013-12-04T09:20:58-08:00" + }, + { + "height": 409572, + "timestamp": "2016-04-30T06:35:43-07:00" + }, + { + "height": 477832, + "timestamp": "2017-07-27T10:23:10-07:00" + }, + { + "height": 534898, + "timestamp": "2018-08-02T13:15:23-07:00" + }, + { + "height": 534747, + "timestamp": "2018-08-01T15:37:52-07:00" + }, + { + "height": 273049, + "timestamp": "2013-12-04T08:15:47-08:00" + }, + { + "height": 477833, + "timestamp": "2017-07-27T10:27:18-07:00" + }, + { + "height": 477841, + "timestamp": "2017-07-27T12:03:49-07:00" + }, + { + "height": 511972, + "timestamp": "2018-03-04T06:25:39-08:00" + }, + { + "height": 533299, + "timestamp": "2018-07-23T08:55:38-07:00" + }, + { + "height": 534901, + "timestamp": "2018-08-02T14:29:22-07:00" + }, + { + "height": 534702, + "timestamp": "2018-08-01T07:15:54-07:00" + }, + { + "height": 534739, + "timestamp": "2018-08-01T14:10:26-07:00" + }, + { + "height": 546097, + "timestamp": "2018-10-16T20:52:28-07:00" + }, + { + "height": 409578, + "timestamp": "2016-04-30T06:55:24-07:00" + }, + { + "height": 534364, + "timestamp": "2018-07-30T00:06:22-07:00" + }, + { + "height": 534698, + "timestamp": "2018-08-01T06:37:12-07:00" + }, + { + "height": 534740, + "timestamp": "2018-08-01T14:16:36-07:00" + }, + { + "height": 409579, + "timestamp": "2016-04-30T07:08:07-07:00" + }, + { + "height": 511967, + "timestamp": "2018-03-04T05:53:30-08:00" + }, + { + "height": 537564, + "timestamp": "2018-08-19T16:27:17-07:00" + }, + { + "height": 534633, + "timestamp": "2018-07-31T19:47:39-07:00" + }, + { + "height": 534759, + "timestamp": "2018-08-01T16:42:56-07:00" + }, + { + "height": 534732, + "timestamp": "2018-08-01T12:40:59-07:00" + }, + { + "height": 534742, + "timestamp": "2018-08-01T14:42:33-07:00" + }, + { + "height": 534692, + "timestamp": "2018-08-01T05:43:04-07:00" + }, + { + "height": 529038, + "timestamp": "2018-06-24T08:02:19-07:00" + }, + { + "height": 537571, + "timestamp": "2018-08-19T17:06:50-07:00" + }, + { + "height": 533303, + "timestamp": "2018-07-23T09:51:22-07:00" + }, + { + "height": 533297, + "timestamp": "2018-07-23T08:43:54-07:00" + }, + { + "height": 534626, + "timestamp": "2018-07-31T18:35:58-07:00" + }, + { + "height": 534764, + "timestamp": "2018-08-01T17:14:46-07:00" + }, + { + "height": 534760, + "timestamp": "2018-08-01T16:45:41-07:00" + }, + { + "height": 534735, + "timestamp": "2018-08-01T13:12:26-07:00" + }, + { + "height": 11, + "timestamp": "2009-01-08T20:12:40-08:00" + }, + { + "height": 534748, + "timestamp": "2018-08-01T15:39:54-07:00" + }, + { + "height": 537565, + "timestamp": "2018-08-19T16:37:35-07:00" + }, + { + "height": 533302, + "timestamp": "2018-07-23T09:50:01-07:00" + }, + { + "height": 534893, + "timestamp": "2018-08-02T12:50:04-07:00" + }, + { + "height": 534738, + "timestamp": "2018-08-01T13:45:07-07:00" + }, + { + "height": 535437, + "timestamp": "2018-08-05T20:39:44-07:00" + }, + { + "height": 534634, + "timestamp": "2018-07-31T20:13:39-07:00" + }, + { + "height": 534766, + "timestamp": "2018-08-01T17:25:16-07:00" + }, + { + "height": 534696, + "timestamp": "2018-08-01T06:21:54-07:00" + }, + { + "height": 534701, + "timestamp": "2018-08-01T07:12:46-07:00" + }, + { + "height": 534695, + "timestamp": "2018-08-01T06:10:36-07:00" + }, + { + "height": 534751, + "timestamp": "2018-08-01T15:57:39-07:00" + }, + { + "height": 534697, + "timestamp": "2018-08-01T06:31:33-07:00" + }, + { + "height": 546101, + "timestamp": "2018-10-16T21:39:58-07:00" + }, + { + "height": 273059, + "timestamp": "2013-12-04T10:11:46-08:00" + }, + { + "height": 477836, + "timestamp": "2017-07-27T11:16:54-07:00" + }, + { + "height": 511969, + "timestamp": "2018-03-04T05:57:00-08:00" + }, + { + "height": 537567, + "timestamp": "2018-08-19T16:56:45-07:00" + }, + { + "height": 534895, + "timestamp": "2018-08-02T13:04:53-07:00" + }, + { + "height": 534765, + "timestamp": "2018-08-01T17:20:59-07:00" + }, + { + "height": 8, + "timestamp": "2009-01-08T19:45:43-08:00" + }, + { + "height": 477840, + "timestamp": "2017-07-27T11:51:16-07:00" + }, + { + "height": 534900, + "timestamp": "2018-08-02T14:20:52-07:00" + }, + { + "height": 534729, + "timestamp": "2018-08-01T12:00:36-07:00" + }, + { + "height": 534737, + "timestamp": "2018-08-01T13:41:56-07:00" + }, + { + "height": 409573, + "timestamp": "2016-04-30T06:37:41-07:00" + }, + { + "height": 535433, + "timestamp": "2018-08-05T19:12:38-07:00" + }, + { + "height": 535434, + "timestamp": "2018-08-05T19:24:21-07:00" + }, + { + "height": 534360, + "timestamp": "2018-07-29T23:22:24-07:00" + }, + { + "height": 534734, + "timestamp": "2018-08-01T13:10:35-07:00" + }, + { + "height": 534726, + "timestamp": "2018-08-01T11:45:10-07:00" + }, + { + "height": 534750, + "timestamp": "2018-08-01T15:55:44-07:00" + }, + { + "height": 534727, + "timestamp": "2018-08-01T11:47:16-07:00" + }, + { + "height": 7, + "timestamp": "2009-01-08T19:39:29-08:00" + }, + { + "height": 273057, + "timestamp": "2013-12-04T09:53:04-08:00" + }, + { + "height": 477842, + "timestamp": "2017-07-27T12:11:49-07:00" + }, + { + "height": 529028, + "timestamp": "2018-06-24T06:47:19-07:00" + }, + { + "height": 537566, + "timestamp": "2018-08-19T16:38:25-07:00" + }, + { + "height": 537562, + "timestamp": "2018-08-19T15:58:05-07:00" + }, + { + "height": 534768, + "timestamp": "2018-08-01T17:29:09-07:00" + }, + { + "height": 273052, + "timestamp": "2013-12-04T08:54:08-08:00" + }, + { + "height": 273056, + "timestamp": "2013-12-04T09:44:34-08:00" + }, + { + "height": 511963, + "timestamp": "2018-03-04T05:32:35-08:00" + }, + { + "height": 511968, + "timestamp": "2018-03-04T05:56:25-08:00" + }, + { + "height": 13, + "timestamp": "2009-01-08T20:23:40-08:00" + }, + { + "height": 546102, + "timestamp": "2018-10-16T21:50:43-07:00" + }, + { + "height": 537561, + "timestamp": "2018-08-19T14:55:35-07:00" + }, + { + "height": 534899, + "timestamp": "2018-08-02T13:42:25-07:00" + }, + { + "height": 534769, + "timestamp": "2018-08-01T17:41:17-07:00" + }, + { + "height": 534746, + "timestamp": "2018-08-01T15:37:21-07:00" + } + ] +} diff --git a/main.go b/main.go index 665ca68..89548d5 100644 --- a/main.go +++ b/main.go @@ -4,12 +4,14 @@ import ( "bufio" "fmt" "github.com/btcsuite/btcutil/hdkeychain" + "github.com/square/beancounter/blockfinder" "log" "math" "net" "os" "strconv" "strings" + "time" "github.com/square/beancounter/accounter" "github.com/square/beancounter/backend" @@ -33,7 +35,7 @@ var ( findAddrN = findAddr.Flag("n", "number of public keys").Short('n').Default("1").Int() findBlock = app.Command("find-block", "Finds the block height for a given date/time.") - findBlockTimestamp = findBlock.Arg("timestamp", "date/time to resolve. E.g. \"2006-01-02 15:04:05 MST\"").Required().String() + findBlockTimestamp = findBlock.Arg("timestamp", "Date/time to resolve. E.g. \"2006-01-02 15:04:05 MST\"").Required().String() findBlockBackend = findBlock.Flag("backend", "electrum | btcd | electrum-recorder | btcd-recorder | fixture").Default("electrum").Enum("electrum", "btcd", "electrum-recorder", "btcd-recorder", "fixture") findBlockAddr = findBlock.Flag("addr", "Backend to connect to initially. Defaults to a hardcoded node for Electrum and localhost for Btcd.").PlaceHolder("HOST:PORT").TCP() findBlockRpcUser = findBlock.Flag("rpcuser", "RPC username").PlaceHolder("USER").String() @@ -66,7 +68,7 @@ func main() { case findAddr.FullCommand(): doFindAddr() case findBlock.FullCommand(): - panic("not yet implemented") + doFindBlock() case computeBalance.FullCommand(): doComputeBalance() default: @@ -166,6 +168,21 @@ func doFindAddr() { fmt.Printf("not found\n") } +func doFindBlock() { + t, err := time.Parse("2006-01-02 15:04:05 MST", *findBlockTimestamp) + PanicOnError(err) + + backend, err := findBlockBuildBackend(Mainnet) + PanicOnError(err) + bf := blockfinder.New(backend) + block, median, timestamp := bf.Search(t) + fmt.Printf("Closest block to '%s' is block #%d with a median time of '%s'\n", + t.String(), block, median.String()) + if *debug { + fmt.Printf("timestamp: '%s'\n", timestamp.String()) + } +} + func doComputeBalance() { err := VerifyMandN(*computeBalanceM, *computeBalanceN) if err != nil { @@ -229,6 +246,57 @@ func doComputeBalance() { fmt.Printf("Balance: %d\n", balance) } +// TODO: copy-pasta +func findBlockBuildBackend(network Network) (backend.Backend, error) { + var b backend.Backend + var err error + switch *findBlockBackend { + case "electrum": + addr, port := getServer(network, *findBlockAddr) + b, err = backend.NewElectrumBackend(addr, port, network) + if err != nil { + return nil, err + } + case "btcd": + b, err = backend.NewBtcdBackend((**findBlockAddr).String(), *findBlockRpcUser, + *findBlockRpcPass, network) + if err != nil { + return nil, err + } + case "electrum-recorder": + if *findBlockFixtureFile == "" { + panic("electrum-recorder backend requires output --fixture-file.") + } + addr, port := getServer(network, *findBlockAddr) + b, err = backend.NewElectrumBackend(addr, port, network) + if err != nil { + return nil, err + } + b, err = backend.NewRecorderBackend(b, *findBlockFixtureFile) + case "btcd-recorder": + if *findBlockFixtureFile == "" { + panic("btcd-recorder backend requires output --fixture-file.") + } + b, err = backend.NewBtcdBackend((*findBlockAddr).String(), *findBlockRpcUser, + *findBlockRpcPass, network) + if err != nil { + return nil, err + } + b, err = backend.NewRecorderBackend(b, *findBlockFixtureFile) + case "fixture": + if *findBlockFixtureFile == "" { + panic("fixture backend requires input --fixture-file.") + } + b, err = backend.NewFixtureBackend(*findBlockFixtureFile) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unreachable") + } + return b, err +} + // TODO: return *backend.Backend, error instead? func computeBalanceBuildBackend(network Network) (backend.Backend, error) { var b backend.Backend