chore!: move blob protos to app under it's own package (#2659)

Ref: celestiaorg/celestia-app#2570

This PR creates a `pkg/blob` where the protos for `Blob` and `BlobTx` is
created. The intention is that this becomes it's own go.mod to avoid
needing to depend on `celestia-core` and our `cosmos-sdk` fork. Other
common structures like namespace should do the same so we can achieve
some cannonical representation of the structs between node and core/app

The reason for creating the types here instead of in `celestia-core` is
so that only `celestia-app` becomes a multi go.mod repo as opposed to
either. Additionally, `Blob` and `BlobTx` were never part of the
upstream CometBFT.

I also made the decision to have a single representation of blobs rather
than a proto and go native. The reason being is one of simplification
and performance. With large blobs, copying the data across to another
type can lead to a noticeable impact. Having a single struct however
does have the dowside of potential coversion errors from uint32 to uint8
so data needs to be validated before it can be used (this is also the
same as before).

This PR also ports over the proofs. I have put this all under a
common/v1 proto package.

app/check_tx.go

@@ -3,10 +3,10 @@ package app
 import (
 	"fmt"
 
+	"github.com/celestiaorg/celestia-app/pkg/blob"
 	blobtypes "github.com/celestiaorg/celestia-app/x/blob/types"
 	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
 	abci "github.com/tendermint/tendermint/abci/types"
-	coretypes "github.com/tendermint/tendermint/types"
 )
 
 // CheckTx implements the ABCI interface and executes a tx in CheckTx mode. This
@@ -15,7 +15,7 @@ import (
 func (app *App) CheckTx(req abci.RequestCheckTx) abci.ResponseCheckTx {
 	tx := req.Tx
 	// check if the transaction contains blobs
-	btx, isBlob := coretypes.UnmarshalBlobTx(tx)
+	btx, isBlob := blob.UnmarshalBlobTx(tx)
 
 	if !isBlob {
 		// reject transactions that can't be decoded

app/process_proposal.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/celestiaorg/celestia-app/app/ante"
+	"github.com/celestiaorg/celestia-app/pkg/blob"
 	"github.com/celestiaorg/celestia-app/pkg/da"
 	"github.com/celestiaorg/celestia-app/pkg/shares"
 	"github.com/celestiaorg/celestia-app/pkg/square"
@@ -16,7 +17,6 @@ import (
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/libs/log"
 	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
-	coretypes "github.com/tendermint/tendermint/types"
 )
 
 const rejectedPropBlockLog = "Rejected proposal block:"
@@ -52,7 +52,7 @@ func (app *App) ProcessProposal(req abci.RequestProcessProposal) (resp abci.Resp
 	// blobTxs have no PFBs present
 	for idx, rawTx := range req.BlockData.Txs {
 		tx := rawTx
-		blobTx, isBlobTx := coretypes.UnmarshalBlobTx(rawTx)
+		blobTx, isBlobTx := blob.UnmarshalBlobTx(rawTx)
 		if isBlobTx {
 			tx = blobTx.Tx
 		}

app/test/check_tx_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/celestiaorg/celestia-app/app"
 	"github.com/celestiaorg/celestia-app/app/encoding"
+	"github.com/celestiaorg/celestia-app/pkg/blob"
 	appns "github.com/celestiaorg/celestia-app/pkg/namespace"
 	"github.com/celestiaorg/celestia-app/pkg/user"
 	testutil "github.com/celestiaorg/celestia-app/test/util"
@@ -82,9 +83,9 @@ func TestCheckTx(t *testing.T) {
 					[]int{100},
 				)[0]
 
-				dtx, _ := coretypes.UnmarshalBlobTx(btx)
+				dtx, _ := blob.UnmarshalBlobTx(btx)
 				dtx.Blobs[0].NamespaceId = appns.RandomBlobNamespace().ID
-				bbtx, err := coretypes.MarshalBlobTx(dtx.Tx, dtx.Blobs[0])
+				bbtx, err := blob.MarshalBlobTx(dtx.Tx, dtx.Blobs[0])
 				require.NoError(t, err)
 				return bbtx
 			},

app/test/integration_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/celestiaorg/celestia-app/app"
 	"github.com/celestiaorg/celestia-app/app/encoding"
 	"github.com/celestiaorg/celestia-app/pkg/appconsts"
+	"github.com/celestiaorg/celestia-app/pkg/blob"
 	"github.com/celestiaorg/celestia-app/pkg/da"
 	appns "github.com/celestiaorg/celestia-app/pkg/namespace"
 	"github.com/celestiaorg/celestia-app/pkg/square"
@@ -32,7 +33,6 @@ import (
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	abci "github.com/tendermint/tendermint/abci/types"
 	tmrand "github.com/tendermint/tendermint/libs/rand"
-	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
 	coretypes "github.com/tendermint/tendermint/types"
 )
 
@@ -210,15 +210,13 @@ func (s *IntegrationTestSuite) TestSubmitPayForBlob() {
 	t := s.T()
 	ns1 := appns.MustNewV0(bytes.Repeat([]byte{1}, appns.NamespaceVersionZeroIDSize))
 
-	mustNewBlob := func(ns appns.Namespace, data []byte, shareVersion uint8) *blobtypes.Blob {
-		b, err := blobtypes.NewBlob(ns, data, shareVersion)
-		require.NoError(t, err)
-		return b
+	mustNewBlob := func(ns appns.Namespace, data []byte, shareVersion uint8) *blob.Blob {
+		return blob.New(ns, data, shareVersion)
 	}
 
 	type test struct {
 		name string
-		blob *blobtypes.Blob
+		blob *blob.Blob
 		opts []user.TxOption
 	}
 
@@ -265,7 +263,7 @@ func (s *IntegrationTestSuite) TestSubmitPayForBlob() {
 			addr := testfactory.GetAddress(s.cctx.Keyring, s.accounts[141])
 			signer, err := user.SetupSigner(s.cctx.GoContext(), s.cctx.Keyring, s.cctx.GRPCClient, addr, s.ecfg)
 			require.NoError(t, err)
-			res, err := signer.SubmitPayForBlob(context.TODO(), []*blobtypes.Blob{tc.blob, tc.blob}, tc.opts...)
+			res, err := signer.SubmitPayForBlob(context.TODO(), []*blob.Blob{tc.blob, tc.blob}, tc.opts...)
 			require.NoError(t, err)
 			require.NotNil(t, res)
 			require.Equal(t, abci.CodeTypeOK, res.Code, res.Logs)
@@ -391,34 +389,34 @@ func (s *IntegrationTestSuite) TestSubmitPayForBlob_blobSizes() {
 
 	type testCase struct {
 		name string
-		blob *tmproto.Blob
+		blob *blob.Blob
 		// txResponseCode is the expected tx response ABCI code.
 		txResponseCode uint32
 	}
 	testCases := []testCase{
 		{
 			name:           "1,000 byte blob",
-			blob:           mustNewBlob(t, 1_000),
+			blob:           mustNewBlob(1_000),
 			txResponseCode: abci.CodeTypeOK,
 		},
 		{
 			name:           "10,000 byte blob",
-			blob:           mustNewBlob(t, 10_000),
+			blob:           mustNewBlob(10_000),
 			txResponseCode: abci.CodeTypeOK,
 		},
 		{
 			name:           "100,000 byte blob",
-			blob:           mustNewBlob(t, 100_000),
+			blob:           mustNewBlob(100_000),
 			txResponseCode: abci.CodeTypeOK,
 		},
 		{
 			name:           "1,000,000 byte blob",
-			blob:           mustNewBlob(t, 1_000_000),
+			blob:           mustNewBlob(1_000_000),
 			txResponseCode: abci.CodeTypeOK,
 		},
 		{
 			name:           "10,000,000 byte blob returns err tx too large",
-			blob:           mustNewBlob(t, 10_000_000),
+			blob:           mustNewBlob(10_000_000),
 			txResponseCode: errors.ErrTxTooLarge.ABCICode(),
 		},
 	}
@@ -427,7 +425,7 @@ func (s *IntegrationTestSuite) TestSubmitPayForBlob_blobSizes() {
 		s.Run(tc.name, func() {
 			subCtx, cancel := context.WithTimeout(s.cctx.GoContext(), 30*time.Second)
 			defer cancel()
-			res, err := signer.SubmitPayForBlob(subCtx, []*blobtypes.Blob{tc.blob}, user.SetGasLimit(1_000_000_000))
+			res, err := signer.SubmitPayForBlob(subCtx, []*blob.Blob{tc.blob}, user.SetGasLimit(1_000_000_000))
 			if tc.txResponseCode == abci.CodeTypeOK {
 				require.NoError(t, err)
 			} else {
@@ -439,10 +437,8 @@ func (s *IntegrationTestSuite) TestSubmitPayForBlob_blobSizes() {
 	}
 }
 
-func mustNewBlob(t *testing.T, blobSize int) *tmproto.Blob {
+func mustNewBlob(blobSize int) *blob.Blob {
 	ns1 := appns.MustNewV0(bytes.Repeat([]byte{1}, appns.NamespaceVersionZeroIDSize))
 	data := tmrand.Bytes(blobSize)
-	result, err := blobtypes.NewBlob(ns1, data, appconsts.ShareVersionZero)
-	require.NoError(t, err)
-	return result
+	return blob.New(ns1, data, appconsts.ShareVersionZero)
 }

app/test/max_total_blob_size_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/celestiaorg/celestia-app/app"
 	"github.com/celestiaorg/celestia-app/app/encoding"
 	"github.com/celestiaorg/celestia-app/pkg/appconsts"
+	"github.com/celestiaorg/celestia-app/pkg/blob"
 	"github.com/celestiaorg/celestia-app/pkg/square"
 	"github.com/celestiaorg/celestia-app/pkg/user"
 	"github.com/celestiaorg/celestia-app/test/util/testfactory"
@@ -17,7 +18,6 @@ import (
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 	abci "github.com/tendermint/tendermint/abci/types"
-	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
 )
 
 const (
@@ -70,24 +70,24 @@ func (s *MaxTotalBlobSizeSuite) TestSubmitPayForBlob_blobSizes() {
 
 	type testCase struct {
 		name string
-		blob *tmproto.Blob
+		blob *blob.Blob
 		// want is the expected tx response ABCI code.
 		want uint32
 	}
 	testCases := []testCase{
 		{
 			name: "1 byte blob",
-			blob: mustNewBlob(t, 1),
+			blob: mustNewBlob(1),
 			want: abci.CodeTypeOK,
 		},
 		{
 			name: "1 mebibyte blob",
-			blob: mustNewBlob(t, mebibyte),
+			blob: mustNewBlob(mebibyte),
 			want: abci.CodeTypeOK,
 		},
 		{
 			name: "2 mebibyte blob",
-			blob: mustNewBlob(t, 2*mebibyte),
+			blob: mustNewBlob(2 * mebibyte),
 			want: blobtypes.ErrTotalBlobSizeTooLarge.ABCICode(),
 		},
 	}
@@ -97,7 +97,7 @@ func (s *MaxTotalBlobSizeSuite) TestSubmitPayForBlob_blobSizes() {
 
 	for _, tc := range testCases {
 		s.Run(tc.name, func() {
-			blobTx, err := signer.CreatePayForBlob([]*tmproto.Blob{tc.blob}, user.SetGasLimit(1e9))
+			blobTx, err := signer.CreatePayForBlob([]*blob.Blob{tc.blob}, user.SetGasLimit(1e9))
 			require.NoError(t, err)
 			subCtx, cancel := context.WithTimeout(s.cctx.GoContext(), 30*time.Second)
 			defer cancel()

app/test/prepare_proposal_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/celestiaorg/celestia-app/app"
 	"github.com/celestiaorg/celestia-app/app/encoding"
 	"github.com/celestiaorg/celestia-app/pkg/appconsts"
+	"github.com/celestiaorg/celestia-app/pkg/blob"
 	appns "github.com/celestiaorg/celestia-app/pkg/namespace"
 	testutil "github.com/celestiaorg/celestia-app/test/util"
 	"github.com/celestiaorg/celestia-app/test/util/blobfactory"
@@ -36,12 +37,8 @@ func TestPrepareProposalPutsPFBsAtEnd(t *testing.T) {
 		testutil.ChainID,
 		accnts[:numBlobTxs],
 		infos[:numBlobTxs],
-		testfactory.Repeat([]*tmproto.Blob{
-			{
-				NamespaceId:  appns.RandomBlobNamespace().ID,
-				Data:         []byte{1},
-				ShareVersion: uint32(appconsts.DefaultShareVersion),
-			},
+		testfactory.Repeat([]*blob.Blob{
+			blob.New(appns.RandomBlobNamespace(), []byte{1}, appconsts.DefaultShareVersion),
 		}, numBlobTxs),
 	)
 

app/test/priority_test.go

@@ -78,7 +78,6 @@ func (s *PriorityTestSuite) TestPriorityByGasPrice() {
 		gasPrice := s.rand.Float64()
 		btx, err := signer.CreatePayForBlob(
 			blobfactory.ManyBlobs(
-				t,
 				s.rand,
 				[]namespace.Namespace{namespace.RandomBlobNamespace()},
 				[]int{100}),

app/test/process_proposal_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/celestiaorg/celestia-app/app"
 	"github.com/celestiaorg/celestia-app/app/encoding"
 	"github.com/celestiaorg/celestia-app/pkg/appconsts"
+	"github.com/celestiaorg/celestia-app/pkg/blob"
 	"github.com/celestiaorg/celestia-app/pkg/da"
 	appns "github.com/celestiaorg/celestia-app/pkg/namespace"
 	"github.com/celestiaorg/celestia-app/pkg/shares"
@@ -112,8 +113,8 @@ func TestProcessProposal(t *testing.T) {
 			name:  "modified a blobTx",
 			input: validData(),
 			mutator: func(d *tmproto.Data) {
-				blobTx, _ := coretypes.UnmarshalBlobTx(blobTxs[0])
-				blobTx.Blobs[0] = &tmproto.Blob{
+				blobTx, _ := blob.UnmarshalBlobTx(blobTxs[0])
+				blobTx.Blobs[0] = &blob.Blob{
 					NamespaceId:      ns1.ID,
 					Data:             data,
 					NamespaceVersion: uint32(ns1.Version),
@@ -128,8 +129,8 @@ func TestProcessProposal(t *testing.T) {
 			name:  "invalid namespace TailPadding",
 			input: validData(),
 			mutator: func(d *tmproto.Data) {
-				blobTx, _ := coretypes.UnmarshalBlobTx(blobTxs[0])
-				blobTx.Blobs[0] = &tmproto.Blob{
+				blobTx, _ := blob.UnmarshalBlobTx(blobTxs[0])
+				blobTx.Blobs[0] = &blob.Blob{
 					NamespaceId:      appns.TailPaddingNamespace.ID,
 					Data:             data,
 					NamespaceVersion: uint32(appns.TailPaddingNamespace.Version),
@@ -144,8 +145,8 @@ func TestProcessProposal(t *testing.T) {
 			name:  "invalid namespace TxNamespace",
 			input: validData(),
 			mutator: func(d *tmproto.Data) {
-				blobTx, _ := coretypes.UnmarshalBlobTx(blobTxs[0])
-				blobTx.Blobs[0] = &tmproto.Blob{
+				blobTx, _ := blob.UnmarshalBlobTx(blobTxs[0])
+				blobTx.Blobs[0] = &blob.Blob{
 					NamespaceId:      appns.TxNamespace.ID,
 					Data:             data,
 					NamespaceVersion: uint32(appns.TxNamespace.Version),
@@ -160,8 +161,8 @@ func TestProcessProposal(t *testing.T) {
 			name:  "invalid namespace ParityShares",
 			input: validData(),
 			mutator: func(d *tmproto.Data) {
-				blobTx, _ := coretypes.UnmarshalBlobTx(blobTxs[0])
-				blobTx.Blobs[0] = &tmproto.Blob{
+				blobTx, _ := blob.UnmarshalBlobTx(blobTxs[0])
+				blobTx.Blobs[0] = &blob.Blob{
 					NamespaceId:      appns.ParitySharesNamespace.ID,
 					Data:             data,
 					NamespaceVersion: uint32(appns.ParitySharesNamespace.Version),
@@ -176,8 +177,8 @@ func TestProcessProposal(t *testing.T) {
 			name:  "invalid blob namespace",
 			input: validData(),
 			mutator: func(d *tmproto.Data) {
-				blobTx, _ := coretypes.UnmarshalBlobTx(blobTxs[0])
-				blobTx.Blobs[0] = &tmproto.Blob{
+				blobTx, _ := blob.UnmarshalBlobTx(blobTxs[0])
+				blobTx.Blobs[0] = &blob.Blob{
 					NamespaceId:      invalidNamespace.ID,
 					Data:             data,
 					ShareVersion:     uint32(appconsts.ShareVersionZero),
@@ -192,7 +193,7 @@ func TestProcessProposal(t *testing.T) {
 			name:  "pfb namespace version does not match blob",
 			input: validData(),
 			mutator: func(d *tmproto.Data) {
-				blobTx, _ := coretypes.UnmarshalBlobTx(blobTxs[0])
+				blobTx, _ := blob.UnmarshalBlobTx(blobTxs[0])
 				blobTx.Blobs[0].NamespaceVersion = appns.NamespaceVersionMax
 				blobTxBytes, _ := blobTx.Marshal()
 				d.Txs[0] = blobTxBytes
@@ -205,8 +206,8 @@ func TestProcessProposal(t *testing.T) {
 			input: validData(),
 			mutator: func(d *tmproto.Data) {
 				index := 4
-				tx, blob := blobfactory.IndexWrappedTxWithInvalidNamespace(t, tmrand.NewRand(), signer, uint32(index))
-				blobTx, err := coretypes.MarshalBlobTx(tx, &blob)
+				tx, b := blobfactory.IndexWrappedTxWithInvalidNamespace(t, tmrand.NewRand(), signer, uint32(index))
+				blobTx, err := blob.MarshalBlobTx(tx, &b)
 				require.NoError(t, err)
 
 				// Replace the data with new contents

app/validate_txs.go

@@ -1,21 +1,21 @@
 package app
 
 import (
+	"github.com/celestiaorg/celestia-app/pkg/blob"
 	"github.com/cosmos/cosmos-sdk/client"
 	"github.com/cosmos/cosmos-sdk/telemetry"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	tmbytes "github.com/tendermint/tendermint/libs/bytes"
 	"github.com/tendermint/tendermint/libs/log"
-	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
 	coretypes "github.com/tendermint/tendermint/types"
 )
 
 // separateTxs decodes raw tendermint txs into normal and blob txs.
-func separateTxs(_ client.TxConfig, rawTxs [][]byte) ([][]byte, []tmproto.BlobTx) {
+func separateTxs(_ client.TxConfig, rawTxs [][]byte) ([][]byte, []blob.BlobTx) {
 	normalTxs := make([][]byte, 0, len(rawTxs))
-	blobTxs := make([]tmproto.BlobTx, 0, len(rawTxs))
+	blobTxs := make([]blob.BlobTx, 0, len(rawTxs))
 	for _, rawTx := range rawTxs {
-		bTx, isBlob := coretypes.UnmarshalBlobTx(rawTx)
+		bTx, isBlob := blob.UnmarshalBlobTx(rawTx)
 		if isBlob {
 			blobTxs = append(blobTxs, bTx)
 		} else {
@@ -69,7 +69,7 @@ func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler
 // filterBlobTxs applies the provided antehandler to each transaction
 // and removes transactions that return an error. Panics are caught by the checkTxValidity
 // function used to apply the ante handler.
-func filterBlobTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler sdk.AnteHandler, txs []tmproto.BlobTx) ([]tmproto.BlobTx, sdk.Context) {
+func filterBlobTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler sdk.AnteHandler, txs []blob.BlobTx) ([]blob.BlobTx, sdk.Context) {
 	n := 0
 	for _, tx := range txs {
 		sdkTx, err := dec(tx.Tx)
@@ -105,11 +105,11 @@ func msgTypes(sdkTx sdk.Tx) []string {
 	return msgNames
 }
 
-func encodeBlobTxs(blobTxs []tmproto.BlobTx) [][]byte {
+func encodeBlobTxs(blobTxs []blob.BlobTx) [][]byte {
 	txs := make([][]byte, len(blobTxs))
 	var err error
 	for i, tx := range blobTxs {
-		txs[i], err = coretypes.MarshalBlobTx(tx.Tx, tx.Blobs...)
+		txs[i], err = blob.MarshalBlobTx(tx.Tx, tx.Blobs...)
 		if err != nil {
 			panic(err)
 		}