Add IFFT Encoding to the High Level Client (#583)

Co-authored-by: Teddy Knox <[email protected]>

api/clients/codecs/blob_codec.go

@@ -0,0 +1,27 @@
+package codecs
+
+import (
+	"fmt"
+)
+
+type BlobEncodingVersion byte
+
+// All blob encodings are IFFT'd before being dispersed
+const (
+	// This minimal blob encoding includes a version byte, a length uint32, and 31 byte field element mapping.
+	DefaultBlobEncoding BlobEncodingVersion = 0x0
+)
+
+type BlobCodec interface {
+	DecodeBlob(encodedData []byte) ([]byte, error)
+	EncodeBlob(rawData []byte) ([]byte, error)
+}
+
+func BlobEncodingVersionToCodec(version BlobEncodingVersion) (BlobCodec, error) {
+	switch version {
+	case DefaultBlobEncoding:
+		return DefaultBlobEncodingCodec{}, nil
+	default:
+		return nil, fmt.Errorf("unsupported blob encoding version: %x", version)
+	}
+}

api/clients/codecs/default_blob_encoding_codec.go

@@ -0,0 +1,49 @@
+package codecs
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/Layr-Labs/eigenda/encoding/utils/codec"
+)
+
+type DefaultBlobEncodingCodec struct{}
+
+var _ BlobCodec = DefaultBlobEncodingCodec{}
+
+func (v DefaultBlobEncodingCodec) EncodeBlob(rawData []byte) ([]byte, error) {
+	// encode blob encoding version byte
+	codecBlobHeader := EncodeCodecBlobHeader(byte(DefaultBlobEncoding), uint32(len(rawData)))
+
+	// encode raw data modulo bn254
+	rawDataPadded := codec.ConvertByPaddingEmptyByte(rawData)
+
+	// append raw data
+	encodedData := append(codecBlobHeader, rawDataPadded...)
+
+	return encodedData, nil
+}
+
+func (v DefaultBlobEncodingCodec) DecodeBlob(encodedData []byte) ([]byte, error) {
+
+	_, length, err := DecodeCodecBlobHeader(encodedData[:32])
+	if err != nil {
+		return nil, err
+	}
+
+	// decode raw data modulo bn254
+	decodedData := codec.RemoveEmptyByteFromPaddedBytes(encodedData[32:])
+
+	// get non blob header data
+	reader := bytes.NewReader(decodedData)
+	rawData := make([]byte, length)
+	n, err := reader.Read(rawData)
+	if err != nil {
+		return nil, fmt.Errorf("failed to copy unpadded data into final buffer, length: %d, bytes read: %d", length, n)
+	}
+	if uint32(n) != length {
+		return nil, fmt.Errorf("data length does not match length prefix")
+	}
+
+	return rawData, nil
+}

api/clients/encodings_test.go → api/clients/codecs/encodings_test.go

@@ -1,10 +1,12 @@
-package clients
+package codecs_test
 
 import (
 	"bytes"
 	"crypto/rand"
 	"math/big"
 	"testing"
+
+	"github.com/Layr-Labs/eigenda/api/clients/codecs"
 )
 
 // Helper function to generate a random byte slice of a given length
@@ -20,7 +22,7 @@ func randomByteSlice(length int64) []byte {
 // TestDefaultBlobEncodingCodec tests the encoding and decoding of random byte streams
 func TestDefaultBlobEncodingCodec(t *testing.T) {
 	// Create an instance of the DefaultBlobEncodingCodec
-	codec := DefaultBlobEncodingCodec{}
+	codec := codecs.DefaultBlobEncodingCodec{}
 
 	// Number of test iterations
 	const iterations = 100
@@ -34,7 +36,10 @@ func TestDefaultBlobEncodingCodec(t *testing.T) {
 		originalData := randomByteSlice(length.Int64() + 1) // ensure it's not length 0
 
 		// Encode the original data
-		encodedData := codec.EncodeBlob(originalData)
+		encodedData, err := codec.EncodeBlob(originalData)
+		if err != nil {
+			t.Fatalf("Iteration %d: failed to encode blob: %v", i, err)
+		}
 
 		// Decode the encoded data
 		decodedData, err := codec.DecodeBlob(encodedData)

api/clients/codecs/utils.go

@@ -0,0 +1,30 @@
+package codecs
+
+import (
+	"encoding/binary"
+	"fmt"
+)
+
+func EncodeCodecBlobHeader(version byte, length uint32) []byte {
+	codecBlobHeader := make([]byte, 32)
+	// first byte is always 0 to ensure the codecBlobHeader is a valid bn254 element
+	// encode version byte
+	codecBlobHeader[1] = version
+
+	// encode length as uint32
+	binary.BigEndian.PutUint32(codecBlobHeader[2:6], length) // uint32 should be more than enough to store the length (approx 4gb)
+	return codecBlobHeader
+}
+
+func DecodeCodecBlobHeader(codecBlobHeader []byte) (byte, uint32, error) {
+	// make sure the codecBlobHeader is 32 bytes long
+	if len(codecBlobHeader) != 32 {
+		err := fmt.Errorf("codecBlobHeader must be exactly 32 bytes long, but got %d bytes", len(codecBlobHeader))
+		return 0, 0, err
+	}
+
+	version := codecBlobHeader[1]
+	length := binary.BigEndian.Uint32(codecBlobHeader[2:6])
+
+	return version, length, nil
+}

api/clients/config.go

@@ -3,6 +3,8 @@ package clients
 import (
 	"fmt"
 	"time"
+
+	"github.com/Layr-Labs/eigenda/api/clients/codecs"
 )
 
 type EigenDAClientConfig struct {
@@ -28,7 +30,13 @@ type EigenDAClientConfig struct {
 	DisableTLS bool
 
 	// The blob encoding version to use when writing blobs from the high level interface.
-	PutBlobEncodingVersion BlobEncodingVersion
+	PutBlobEncodingVersion codecs.BlobEncodingVersion
+
+	// Point verification mode does an IFFT on data before it is written, and does an FFT on data after it is read.
+	// This makes it possible to open points on the KZG commitment to prove that the field elements correspond to
+	// the commitment. With this mode disabled, you will need to supply the entire blob to perform a verification
+	// that any part of the data matches the KZG commitment.
+	DisablePointVerificationMode bool
 }
 
 func (c *EigenDAClientConfig) CheckAndSetDefaults() error {

api/clients/eigenda_client.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"time"
 
+	"github.com/Layr-Labs/eigenda/api/clients/codecs"
 	grpcdisperser "github.com/Layr-Labs/eigenda/api/grpc/disperser"
 	"github.com/Layr-Labs/eigenda/core/auth"
 	"github.com/Layr-Labs/eigenda/disperser"
@@ -23,7 +24,7 @@ type EigenDAClient struct {
 	Config   EigenDAClientConfig
 	Log      log.Logger
 	Client   DisperserClient
-	PutCodec BlobCodec
+	PutCodec codecs.BlobCodec
 }
 
 var _ IEigenDAClient = EigenDAClient{}
@@ -43,7 +44,7 @@ func NewEigenDAClient(log log.Logger, config EigenDAClientConfig) (*EigenDAClien
 	llConfig := NewConfig(host, port, config.ResponseTimeout, !config.DisableTLS)
 	llClient := NewDisperserClient(llConfig, signer)
 
-	codec, err := BlobEncodingVersionToCodec(config.PutBlobEncodingVersion)
+	codec, err := codecs.BlobEncodingVersionToCodec(config.PutBlobEncodingVersion)
 	if err != nil {
 		return nil, fmt.Errorf("error initializing EigenDA client: %w", err)
 	}
@@ -66,18 +67,31 @@ func (m EigenDAClient) GetBlob(ctx context.Context, BatchHeaderHash []byte, Blob
 		return nil, fmt.Errorf("blob has length zero")
 	}
 
-	version := BlobEncodingVersion(data[0])
-	codec, err := BlobEncodingVersionToCodec(version)
+	if !m.Config.DisablePointVerificationMode {
+		// we assume the data is already IFFT'd so we FFT it before decoding
+		data, err = FFT(data)
+		if err != nil {
+			return nil, fmt.Errorf("error FFTing data: %w", err)
+		}
+	}
+
+	// get version and length from codec blob header
+	version, _, err := codecs.DecodeCodecBlobHeader(data[:32])
+	if err != nil {
+		return nil, fmt.Errorf("error getting blob: %w", err)
+	}
+
+	codec, err := codecs.BlobEncodingVersionToCodec(codecs.BlobEncodingVersion(version))
 	if err != nil {
 		return nil, fmt.Errorf("error getting blob: %w", err)
 	}
 
-	rawData, err := codec.DecodeBlob(data)
+	decodedData, err := codec.DecodeBlob(data)
 	if err != nil {
 		return nil, fmt.Errorf("error getting blob: %w", err)
 	}
 
-	return rawData, nil
+	return decodedData, nil
 }
 
 func (m EigenDAClient) PutBlob(ctx context.Context, data []byte) (*grpcdisperser.BlobInfo, error) {
@@ -105,13 +119,25 @@ func (m EigenDAClient) putBlob(ctx context.Context, rawData []byte, resultChan c
 		errChan <- fmt.Errorf("PutCodec cannot be nil")
 		return
 	}
-	data := m.PutCodec.EncodeBlob(rawData)
+	data, err := m.PutCodec.EncodeBlob(rawData)
+	if err != nil {
+		errChan <- fmt.Errorf("error encoding blob: %w", err)
+		return
+	}
+
+	if !m.Config.DisablePointVerificationMode {
+		// convert data to fr.Element
+		data, err = IFFT(data)
+		if err != nil {
+			errChan <- fmt.Errorf("error IFFTing data: %w", err)
+			return
+		}
+	}
 
 	customQuorumNumbers := make([]uint8, len(m.Config.CustomQuorumIDs))
 	for i, e := range m.Config.CustomQuorumIDs {
 		customQuorumNumbers[i] = uint8(e)
 	}
-
 	// disperse blob
 	blobStatus, requestID, err := m.Client.DisperseBlobAuthenticated(ctx, data, customQuorumNumbers)
 	if err != nil {