Port BlobClient from old 4844 branch

This ports BlobClient from the eip-4844-experimental branch, with the
prysm dependency removed (relevant code copied to util/beaconclient)
and the kZGToVersionedHash function copied from geth rather than
modifying our fork to make it public as I had done before, since it is
so simple.

"A little copying is better than a little dependency."
- Rob Pike, Go Proverbs

arbnode/blob_reader.go

@@ -0,0 +1,185 @@
+// Copyright 2023, Offchain Labs, Inc.
+// For license information, see https://github.com/nitro/blob/master/LICENSE
+
+package arbnode
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto/kzg4844"
+	"github.com/offchainlabs/nitro/arbutil"
+	"github.com/offchainlabs/nitro/util/beaconclient"
+	"github.com/offchainlabs/nitro/util/pretty"
+	"github.com/pkg/errors"
+
+	"github.com/spf13/pflag"
+)
+
+type BlobClient struct {
+	bc *beaconclient.Client
+	ec arbutil.L1Interface
+
+	// The genesis time time won't change so only request it once.
+	cachedGenesisTime uint64
+}
+
+type BlobClientConfig struct {
+	BeaconChainUrl string `koanf:"beacon-chain-url"`
+}
+
+var DefaultBlobClientConfig = BlobClientConfig{
+	BeaconChainUrl: "",
+}
+
+func BlobClientAddOptions(prefix string, f *pflag.FlagSet) {
+	f.String(prefix+".beacon-chain-url", DefaultBlobClientConfig.BeaconChainUrl, "Beacon Chain url to use for fetching blobs")
+}
+
+func NewBlobClient(bc *beaconclient.Client, ec arbutil.L1Interface) *BlobClient {
+	return &BlobClient{bc: bc, ec: ec}
+}
+
+// Get all the blobs associated with a particular block.
+func (b *BlobClient) GetBlobs(ctx context.Context, blockHash common.Hash, versionedHashes []common.Hash) ([]kzg4844.Blob, error) {
+	header, err := b.ec.HeaderByHash(ctx, blockHash)
+	if err != nil {
+		return nil, err
+	}
+
+	genesisTime, err := b.genesisTime(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO make denominator configurable for devnets with faster block time
+	slot := (header.Time - genesisTime) / 12
+
+	return b.blobSidecars(ctx, slot, versionedHashes)
+}
+
+type blobResponse struct {
+	Data []blobResponseItem `json:"data"`
+}
+type blobResponseItem struct {
+	BlockRoot       string `json:"block_root"`
+	Index           int    `json:"index"`
+	Slot            uint64 `json:"slot"`
+	BlockParentRoot string `json:"block_parent_root"`
+	ProposerIndex   uint64 `json:"proposer_index"`
+	Blob            string `json:"blob"`
+	KzgCommitment   string `json:"kzg_commitment"`
+	KzgProof        string `json:"kzg_proof"`
+}
+
+func (b *BlobClient) blobSidecars(ctx context.Context, slot uint64, versionedHashes []common.Hash) ([]kzg4844.Blob, error) {
+	body, err := b.bc.Get(ctx, fmt.Sprintf("/eth/v1/beacon/blob_sidecars/%d", slot))
+	if err != nil {
+		return nil, errors.Wrap(err, "error calling beacon client in blobSidecars")
+	}
+
+	br := &blobResponse{}
+	err = json.Unmarshal(body, br)
+	if err != nil {
+		return nil, errors.Wrap(err, "error decoding json response in blobSidecars")
+	}
+
+	if len(br.Data) == 0 {
+		return nil, fmt.Errorf("no blobs found for slot %d", slot)
+	}
+
+	blobs := make([]kzg4844.Blob, len(versionedHashes))
+	var totalFound int
+
+	for i := range blobs {
+		commitmentBytes, err := hexutil.Decode(br.Data[i].KzgCommitment)
+		if err != nil {
+			return nil, fmt.Errorf("couldn't decode commitment for slot(%d) at index(%d), commitment(%s)", slot, br.Data[i].Index, pretty.FirstFewChars(br.Data[i].KzgCommitment))
+		}
+		var commitment kzg4844.Commitment
+		copy(commitment[:], commitmentBytes)
+		versionedHash := kZGToVersionedHash(commitment)
+
+		// The versioned hashes of the blob commitments are produced in the by HASH_OPCODE_BYTE,
+		// presumably in the order they were added to the tx. The spec is unclear if the blobs
+		// need to be returned in any particular order from the beacon API, so we put them back in
+		// the order from the tx.
+		var j int
+		var found bool
+		for j = range versionedHashes {
+			if versionedHashes[j] == versionedHash {
+				found = true
+				totalFound++
+				break
+			}
+		}
+		if !found {
+			continue
+		}
+
+		blob, err := hexutil.Decode(br.Data[i].Blob)
+		if err != nil {
+			return nil, fmt.Errorf("couldn't decode blob for slot(%d) at index(%d), blob(%s)", slot, br.Data[i].Index, pretty.FirstFewChars(br.Data[i].Blob))
+		}
+		copy(blobs[j][:], blob)
+
+		proofBytes, err := hexutil.Decode(br.Data[i].KzgProof)
+		if err != nil {
+			return nil, fmt.Errorf("couldn't decode proof for slot(%d) at index(%d), proof(%s)", slot, br.Data[i].Index, pretty.FirstFewChars(br.Data[i].KzgProof))
+		}
+		var proof kzg4844.Proof
+		copy(proof[:], proofBytes)
+
+		err = kzg4844.VerifyBlobProof(blobs[j], commitment, proof)
+		if err != nil {
+			return nil, fmt.Errorf("failed to verify blob proof for blob at slot(%d) at index(%d), blob(%s)", slot, br.Data[i].Index, pretty.FirstFewChars(br.Data[i].Blob))
+		}
+	}
+
+	if totalFound < len(versionedHashes) {
+		return nil, fmt.Errorf("not all of the requested blobs (%d/%d) were found at slot (%d), can't reconstruct batch payload", totalFound, len(versionedHashes), slot)
+	}
+
+	return blobs, nil
+}
+
+type genesisResponse struct {
+	GenesisTime uint64 `json:"genesis_time"`
+	// don't currently care about other fields, add if needed
+}
+
+func (b *BlobClient) genesisTime(ctx context.Context) (uint64, error) {
+	if b.cachedGenesisTime > 0 {
+		return b.cachedGenesisTime, nil
+	}
+
+	body, err := b.bc.Get(ctx, "/eth/v1/beacon/genesis")
+	if err != nil {
+		return 0, errors.Wrap(err, "error calling beacon client in genesisTime")
+	}
+
+	gr := &genesisResponse{}
+	dataWrapper := &struct{ Data *genesisResponse }{Data: gr}
+	err = json.Unmarshal(body, dataWrapper)
+	if err != nil {
+		return 0, errors.Wrap(err, "error decoding json response in genesisTime")
+	}
+
+	return gr.GenesisTime, nil
+}
+
+// The following code is taken from core/vm/contracts.go
+const (
+	blobCommitmentVersionKZG uint8 = 0x01 // Version byte for the point evaluation precompile.
+)
+
+func kZGToVersionedHash(kzg kzg4844.Commitment) common.Hash {
+	h := sha256.Sum256(kzg[:])
+	h[0] = blobCommitmentVersionKZG
+
+	return h
+}

go.mod

@@ -36,6 +36,7 @@ require (
 	github.com/libp2p/go-libp2p v0.27.8
 	github.com/multiformats/go-multiaddr v0.9.0
 	github.com/multiformats/go-multihash v0.2.1
+	github.com/pkg/errors v0.9.1
 	github.com/r3labs/diff/v3 v3.0.1
 	github.com/rivo/tview v0.0.0-20230814110005-ccc2c8119703
 	github.com/spf13/pflag v1.0.5
@@ -233,7 +234,6 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/openzipkin/zipkin-go v0.4.0 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/polydawn/refmt v0.89.0 // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect

util/beaconclient/client.go

@@ -0,0 +1,98 @@
+package beaconclient
+
+import (
+	"context"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+
+	"github.com/pkg/errors"
+)
+
+// Client is a wrapper object around the HTTP client.
+// Taken from prysm/api/client.
+type Client struct {
+	hc      *http.Client
+	baseURL *url.URL
+	token   string
+}
+
+// NewClient constructs a new client with the provided options (ex WithTimeout).
+// `host` is the base host + port used to construct request urls. This value can be
+// a URL string, or NewClient will assume an http endpoint if just `host:port` is used.
+func NewClient(host string, opts ...ClientOpt) (*Client, error) {
+	u, err := urlForHost(host)
+	if err != nil {
+		return nil, err
+	}
+	c := &Client{
+		hc:      &http.Client{},
+		baseURL: u,
+	}
+	for _, o := range opts {
+		o(c)
+	}
+	return c, nil
+}
+
+// Token returns the bearer token used for jwt authentication
+func (c *Client) Token() string {
+	return c.token
+}
+
+// BaseURL returns the base url of the client
+func (c *Client) BaseURL() *url.URL {
+	return c.baseURL
+}
+
+// Do execute the request against the http client
+func (c *Client) Do(req *http.Request) (*http.Response, error) {
+	return c.hc.Do(req)
+}
+
+func urlForHost(h string) (*url.URL, error) {
+	// try to parse as url (being permissive)
+	u, err := url.Parse(h)
+	if err == nil && u.Host != "" {
+		return u, nil
+	}
+	// try to parse as host:port
+	host, port, err := net.SplitHostPort(h)
+	if err != nil {
+		return nil, ErrMalformedHostname
+	}
+	return &url.URL{Host: net.JoinHostPort(host, port), Scheme: "http"}, nil
+}
+
+// NodeURL returns a human-readable string representation of the beacon node base url.
+func (c *Client) NodeURL() string {
+	return c.baseURL.String()
+}
+
+// Get is a generic, opinionated GET function to reduce boilerplate amongst the getters in this package.
+func (c *Client) Get(ctx context.Context, path string, opts ...ReqOption) ([]byte, error) {
+	u := c.baseURL.ResolveReference(&url.URL{Path: path})
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	for _, o := range opts {
+		o(req)
+	}
+	r, err := c.hc.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		err = r.Body.Close()
+	}()
+	if r.StatusCode != http.StatusOK {
+		return nil, Non200Err(r)
+	}
+	b, err := io.ReadAll(r.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "error reading http response body")
+	}
+	return b, nil
+}

util/beaconclient/errors.go

@@ -0,0 +1,40 @@
+package beaconclient
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/pkg/errors"
+)
+
+// ErrMalformedHostname is used to indicate if a host name's format is incorrect.
+var ErrMalformedHostname = errors.New("hostname must include port, separated by one colon, like example.com:3500")
+
+// ErrNotOK is used to indicate when an HTTP request to the API failed with any non-2xx response code.
+// More specific errors may be returned, but an error in reaction to a non-2xx response will always wrap ErrNotOK.
+var ErrNotOK = errors.New("did not receive 2xx response from API")
+
+// ErrNotFound specifically means that a '404 - NOT FOUND' response was received from the API.
+var ErrNotFound = errors.Wrap(ErrNotOK, "recv 404 NotFound response from API")
+
+// ErrInvalidNodeVersion indicates that the /eth/v1/node/version API response format was not recognized.
+var ErrInvalidNodeVersion = errors.New("invalid node version response")
+
+// Non200Err is a function that parses an HTTP response to handle responses that are not 200 with a formatted error.
+func Non200Err(response *http.Response) error {
+	bodyBytes, err := io.ReadAll(response.Body)
+	var body string
+	if err != nil {
+		body = "(Unable to read response body.)"
+	} else {
+		body = "response body:\n" + string(bodyBytes)
+	}
+	msg := fmt.Sprintf("code=%d, url=%s, body=%s", response.StatusCode, response.Request.URL, body)
+	switch response.StatusCode {
+	case 404:
+		return errors.Wrap(ErrNotFound, msg)
+	default:
+		return errors.Wrap(ErrNotOK, msg)
+	}
+}

util/beaconclient/options.go

@@ -0,0 +1,48 @@
+package beaconclient
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+)
+
+// ReqOption is a request functional option.
+type ReqOption func(*http.Request)
+
+// WithSSZEncoding is a request functional option that adds SSZ encoding header.
+func WithSSZEncoding() ReqOption {
+	return func(req *http.Request) {
+		req.Header.Set("Accept", "application/octet-stream")
+	}
+}
+
+// WithAuthorizationToken is a request functional option that adds header for authorization token.
+func WithAuthorizationToken(token string) ReqOption {
+	return func(req *http.Request) {
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+	}
+}
+
+// ClientOpt is a functional option for the Client type (http.Client wrapper)
+type ClientOpt func(*Client)
+
+// WithTimeout sets the .Timeout attribute of the wrapped http.Client.
+func WithTimeout(timeout time.Duration) ClientOpt {
+	return func(c *Client) {
+		c.hc.Timeout = timeout
+	}
+}
+
+// WithRoundTripper replaces the underlying HTTP's transport with a custom one.
+func WithRoundTripper(t http.RoundTripper) ClientOpt {
+	return func(c *Client) {
+		c.hc.Transport = t
+	}
+}
+
+// WithAuthenticationToken sets an oauth token to be used.
+func WithAuthenticationToken(token string) ClientOpt {
+	return func(c *Client) {
+		c.token = token
+	}
+}