feat!: reserve the last 256 namespaces for protocol use (#2257)

Closes #2253

(cherry picked from commit cb8247e)

# Conflicts:
#	pkg/namespace/consts.go
#	pkg/namespace/namespace.go
#	pkg/namespace/namespace_test.go
#	pkg/shares/padding.go
#	specs/src/specs/namespace.md

pkg/namespace/consts.go

@@ -36,35 +36,63 @@ var (
 	// NamespaceVersionZeroPrefix is the prefix of a namespace ID for version 0.
 	NamespaceVersionZeroPrefix = bytes.Repeat([]byte{0}, NamespaceVersionZeroPrefixSize)
 
-	// TxNamespace is the namespace reserved for transaction data.
-	TxNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 1})
+	// TxNamespace is the namespace reserved for ordinary Cosmos SDK transactions.
+	TxNamespace = primaryReservedNamespace(0x01)
 
 	// IntermediateStateRootsNamespace is the namespace reserved for
 	// intermediate state root data.
-	IntermediateStateRootsNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 2})
+	IntermediateStateRootsNamespace = primaryReservedNamespace(0x02)
 
 	// PayForBlobNamespace is the namespace reserved for PayForBlobs transactions.
-	PayForBlobNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 4})
+	PayForBlobNamespace = primaryReservedNamespace(0x04)
 
-	// ReservedPaddingNamespace is the namespace used for padding after all
-	// reserved namespaces. In practice this padding is after transactions
-	// (ordinary and PFBs) but before blobs.
-	ReservedPaddingNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255})
+	// PrimaryReservedPaddingNamespace is the namespace used for padding after all
+	// primary reserved namespaces.
+	PrimaryReservedPaddingNamespace = primaryReservedNamespace(0xFF)
 
+<<<<<<< HEAD
 	// MaxReservedNamespace is lexicographically the largest namespace that is
 	// reserved for protocol use.
 	MaxReservedNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255})
+=======
+	// MaxPrimaryReservedNamespace is the highest primary reserved namespace.
+	// Namespaces lower than this are reserved for protocol use.
+	MaxPrimaryReservedNamespace = primaryReservedNamespace(0xFF)
+
+	// MinSecondaryReservedNamespace is the lowest secondary reserved namespace
+	// reserved for protocol use. Namespaces higher than this are reserved for
+	// protocol use.
+	MinSecondaryReservedNamespace = secondaryReservedNamespace(0x00)
+>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257))
 
 	// TailPaddingNamespace is the namespace reserved for tail padding. All data
 	// with this namespace will be ignored.
-	TailPaddingNamespace = Namespace{
-		Version: math.MaxUint8,
-		ID:      append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), 0xFE),
-	}
+	TailPaddingNamespace = secondaryReservedNamespace(0xFE)
 
 	// ParitySharesNamespace is the namespace reserved for erasure coded data.
+<<<<<<< HEAD
 	ParitySharesNamespace = Namespace{
 		Version: math.MaxUint8,
 		ID:      bytes.Repeat([]byte{0xFF}, NamespaceIDSize),
 	}
+=======
+	ParitySharesNamespace = secondaryReservedNamespace(0xFF)
+
+	// SupportedBlobNamespaceVersions is a list of namespace versions that can be specified by a user for blobs.
+	SupportedBlobNamespaceVersions = []uint8{NamespaceVersionZero}
+>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257))
 )
+
+func primaryReservedNamespace(lastByte byte) Namespace {
+	return Namespace{
+		Version: NamespaceVersionZero,
+		ID:      append(bytes.Repeat([]byte{0x00}, NamespaceIDSize-1), lastByte),
+	}
+}
+
+func secondaryReservedNamespace(lastByte byte) Namespace {
+	return Namespace{
+		Version: NamespaceVersionMax,
+		ID:      append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), lastByte),
+	}
+}

pkg/namespace/namespace.go

@@ -122,7 +122,19 @@ func validateID(version uint8, id []byte) error {
 }
 
 func (n Namespace) IsReserved() bool {
+<<<<<<< HEAD
 	return bytes.Compare(n.Bytes(), MaxReservedNamespace.Bytes()) < 1
+=======
+	return n.IsPrimaryReserved() || n.IsSecondaryReserved()
+}
+
+func (n Namespace) IsPrimaryReserved() bool {
+	return n.IsLessOrEqualThan(MaxPrimaryReservedNamespace)
+}
+
+func (n Namespace) IsSecondaryReserved() bool {
+	return n.IsGreaterOrEqualThan(MinSecondaryReservedNamespace)
+>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257))
 }
 
 func (n Namespace) IsParityShares() bool {
@@ -133,8 +145,8 @@ func (n Namespace) IsTailPadding() bool {
 	return bytes.Equal(n.Bytes(), TailPaddingNamespace.Bytes())
 }
 
-func (n Namespace) IsReservedPadding() bool {
-	return bytes.Equal(n.Bytes(), ReservedPaddingNamespace.Bytes())
+func (n Namespace) IsPrimaryReservedPadding() bool {
+	return bytes.Equal(n.Bytes(), PrimaryReservedPaddingNamespace.Bytes())
 }
 
 func (n Namespace) IsTx() bool {

pkg/namespace/namespace_test.go

@@ -2,6 +2,11 @@ package namespace
 
 import (
 	"bytes"
+<<<<<<< HEAD
+=======
+	"math"
+	"reflect"
+>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257))
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -195,3 +200,93 @@ func TestBytes(t *testing.T) {
 
 	assert.Equal(t, want, got)
 }
+<<<<<<< HEAD
+=======
+
+func TestLeftPad(t *testing.T) {
+	tests := []struct {
+		input    []byte
+		size     int
+		expected []byte
+	}{
+		// input smaller than pad size
+		{[]byte{1, 2, 3}, 10, []byte{0, 0, 0, 0, 0, 0, 0, 1, 2, 3}},
+		{[]byte{1}, 5, []byte{0, 0, 0, 0, 1}},
+		{[]byte{1, 2}, 4, []byte{0, 0, 1, 2}},
+
+		// input equal to pad size
+		{[]byte{1, 2, 3}, 3, []byte{1, 2, 3}},
+		{[]byte{1, 2, 3, 4}, 4, []byte{1, 2, 3, 4}},
+
+		// input larger than pad size
+		{[]byte{1, 2, 3, 4, 5}, 4, []byte{1, 2, 3, 4, 5}},
+		{[]byte{1, 2, 3, 4, 5, 6, 7}, 3, []byte{1, 2, 3, 4, 5, 6, 7}},
+
+		// input size 0
+		{[]byte{}, 8, []byte{0, 0, 0, 0, 0, 0, 0, 0}},
+		{[]byte{}, 0, []byte{}},
+	}
+
+	for _, test := range tests {
+		result := leftPad(test.input, test.size)
+		assert.True(t, reflect.DeepEqual(result, test.expected))
+	}
+}
+
+func TestIsReserved(t *testing.T) {
+	type testCase struct {
+		ns   Namespace
+		want bool
+	}
+	testCases := []testCase{
+		{
+			ns:   MustNewV0(bytes.Repeat([]byte{1}, NamespaceVersionZeroIDSize)),
+			want: false,
+		},
+		{
+			ns:   TxNamespace,
+			want: true,
+		},
+		{
+			ns:   IntermediateStateRootsNamespace,
+			want: true,
+		},
+		{
+			ns:   PayForBlobNamespace,
+			want: true,
+		},
+		{
+			ns:   PrimaryReservedPaddingNamespace,
+			want: true,
+		},
+		{
+			ns:   MaxPrimaryReservedNamespace,
+			want: true,
+		},
+		{
+			ns:   MinSecondaryReservedNamespace,
+			want: true,
+		},
+		{
+			ns:   TailPaddingNamespace,
+			want: true,
+		},
+		{
+			ns:   ParitySharesNamespace,
+			want: true,
+		},
+		{
+			ns: Namespace{
+				Version: math.MaxUint8,
+				ID:      append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), 1),
+			},
+			want: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		got := tc.ns.IsReserved()
+		assert.Equal(t, tc.want, got)
+	}
+}
+>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257))

pkg/shares/padding.go

@@ -52,7 +52,11 @@ func NamespacePaddingShares(ns appns.Namespace, n int) ([]Share, error) {
 // first blob can start at an index that conforms to non-interactive default
 // rules.
 func ReservedPaddingShare() Share {
+<<<<<<< HEAD
 	share, err := NamespacePaddingShare(appns.ReservedPaddingNamespace)
+=======
+	share, err := NamespacePaddingShare(appns.PrimaryReservedPaddingNamespace, appconsts.ShareVersionZero)
+>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257))
 	if err != nil {
 		panic(err)
 	}
@@ -61,7 +65,11 @@ func ReservedPaddingShare() Share {
 
 // ReservedPaddingShare returns n reserved padding shares.
 func ReservedPaddingShares(n int) []Share {
+<<<<<<< HEAD
 	shares, err := NamespacePaddingShares(appns.ReservedPaddingNamespace, n)
+=======
+	shares, err := NamespacePaddingShares(appns.PrimaryReservedPaddingNamespace, appconsts.ShareVersionZero, n)
+>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257))
 	if err != nil {
 		panic(err)
 	}

pkg/shares/padding_test.go

@@ -23,7 +23,7 @@ var nsOnePadding, _ = zeroPadIfNecessary(
 
 var reservedPadding, _ = zeroPadIfNecessary(
 	append(
-		appns.ReservedPaddingNamespace.Bytes(),
+		appns.PrimaryReservedPaddingNamespace.Bytes(),
 		[]byte{
 			1,          // info byte
 			0, 0, 0, 0, // sequence len

pkg/shares/parse_test.go

@@ -162,11 +162,11 @@ func TestParseShares(t *testing.T) {
 			ignorePadding: false,
 			want: []ShareSequence{
 				{
-					Namespace: appns.ReservedPaddingNamespace,
+					Namespace: appns.PrimaryReservedPaddingNamespace,
 					Shares:    []Share{ReservedPaddingShare()},
 				},
 				{
-					Namespace: appns.ReservedPaddingNamespace,
+					Namespace: appns.PrimaryReservedPaddingNamespace,
 					Shares:    []Share{ReservedPaddingShare()},
 				},
 			},

pkg/shares/share_sequence_test.go

@@ -159,7 +159,7 @@ func Test_validSequenceLen(t *testing.T) {
 	}
 
 	reservedPadding := ShareSequence{
-		Namespace: appns.ReservedPaddingNamespace,
+		Namespace: appns.PrimaryReservedPaddingNamespace,
 		Shares:    []Share{ReservedPaddingShare()},
 	}
 

pkg/shares/shares.go

@@ -121,11 +121,11 @@ func (s *Share) IsPadding() (bool, error) {
 	if err != nil {
 		return false, err
 	}
-	isReservedPadding, err := s.isReservedPadding()
+	isPrimaryReservedPadding, err := s.isPrimaryReservedPadding()
 	if err != nil {
 		return false, err
 	}
-	return isNamespacePadding || isTailPadding || isReservedPadding, nil
+	return isNamespacePadding || isTailPadding || isPrimaryReservedPadding, nil
 }
 
 func (s *Share) isNamespacePadding() (bool, error) {
@@ -149,12 +149,12 @@ func (s *Share) isTailPadding() (bool, error) {
 	return ns.IsTailPadding(), nil
 }
 
-func (s *Share) isReservedPadding() (bool, error) {
+func (s *Share) isPrimaryReservedPadding() (bool, error) {
 	ns, err := s.Namespace()
 	if err != nil {
 		return false, err
 	}
-	return ns.IsReservedPadding(), nil
+	return ns.IsPrimaryReservedPadding(), nil
 }
 
 func (s *Share) ToBytes() []byte {

specs/src/specs/namespace.md

@@ -42,6 +42,7 @@ The namespace ID is a 28 byte identifier that uniquely identifies a namespace. T
 
 ## Reserved Namespaces
 
+<<<<<<< HEAD
 | name                                | type        | value                                                          | description                                                                                          |
 |-------------------------------------|-------------|----------------------------------------------------------------|------------------------------------------------------------------------------------------------------|
 | `TRANSACTION_NAMESPACE`             | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000001` | Transactions: requests that modify the state.                                                        |
@@ -51,6 +52,33 @@ The namespace ID is a 28 byte identifier that uniquely identifies a namespace. T
 | `MAX_RESERVED_NAMESPACE`            | `Namespace` | `0x00000000000000000000000000000000000000000000000000000000FF` | Max reserved namespace is lexicographically the largest namespace that is reserved for protocol use. |
 | `TAIL_PADDING_NAMESPACE`            | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Tail padding for blobs: padding after all blobs to fill up the original data square.                 |
 | `PARITY_SHARE_NAMESPACE`            | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Parity shares: extended shares in the available data matrix.                                         |
+=======
+Celestia reserves some namespaces for protocol use.
+These namespaces are called "reserved namespaces".
+Reserved namespaces are used to arrange the contents of the [data square](./data_square_layout.md).
+Applications MUST NOT use reserved namespaces for their blob data.
+Reserved namespaces fall into two categories: _Primary_ and _Secondary_.
+
+- Primary: Namespaces with values less than or equal to `0x00000000000000000000000000000000000000000000000000000000FF`. Primary namespaces always have a version of `0`.
+- Secondary: Namespaces with values greater than or equal to `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00`.
+Secondary namespaces always have a version of `255` (`0xFF`) so that they are placed after all user specifiable namespaces in a sorted data square.
+The `PARITY_SHARE_NAMESPACE` uses version `255` (`0xFF`) to enable more efficient proof generation within the context of [nmt](https://github.com/celestiaorg/nmt), where it is used in conjunction with the `IgnoreMaxNamespace` feature.
+The `TAIL_PADDING_NAMESPACE` uses the version `255` to ensure that padding shares are always placed at the end of the Celestia data square even if a new user-specifiable version is introduced.
+
+Below is a list of the current reserved namespaces.
+For additional information on the significance and application of the reserved namespaces, please refer to the [Data Square Layout](./data_square_layout.md) specifications.
+
+| name                                 | type        | category  | value                                                          | description                                                                |
+|--------------------------------------|-------------|-----------|----------------------------------------------------------------|----------------------------------------------------------------------------|
+| `TRANSACTION_NAMESPACE`              | `Namespace` | Primary   | `0x0000000000000000000000000000000000000000000000000000000001` | Namespace for ordinary Cosmos SDK transactions.                            |
+| `INTERMEDIATE_STATE_ROOT_NAMESPACE`  | `Namespace` | Primary   | `0x0000000000000000000000000000000000000000000000000000000002` | Namespace for intermediate state roots (not currently utilized).           |
+| `PAY_FOR_BLOB_NAMESPACE`             | `Namespace` | Primary   | `0x0000000000000000000000000000000000000000000000000000000004` | Namespace for transactions that contain a PayForBlob.                      |
+| `PRIMARY_RESERVED_PADDING_NAMESPACE` | `Namespace` | Primary   | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for padding after all primary reserved namespaces.               |
+| `MAX_PRIMARY_RESERVED_NAMESPACE`     | `Namespace` | Primary   | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for the highest primary reserved namespace.                      |
+| `MIN_SECONDARY_RESERVED_NAMESPACE`   | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00` | Namespace for the lowest secondary reserved namespace.                     |
+| `TAIL_PADDING_NAMESPACE`             | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Namespace for padding after all blobs to fill up the original data square. |
+| `PARITY_SHARE_NAMESPACE`             | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Namespace for parity shares.                                               |
+>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257))
 
 ## Assumptions and Considerations
 

specs/src/specs/shares.md

@@ -18,7 +18,7 @@ User submitted transactions are split into shares (see [share splitting](#share-
 
 [Padding](#padding) shares are added to the `k * k` matrix to ensure:
 
-1. Blob sequences start on an index that conforms to [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) (see [namespace padding share](#namespace-padding-share) and [reserved padding share](#reserved-padding-share))
+1. Blob sequences start on an index that conforms to [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) (see [namespace padding share](#namespace-padding-share) and [reserved padding share](#primary-reserved-padding-share))
 1. The number of shares in the matrix is a perfect square (see [tail padding share](#tail-padding-share))
 
 ## Share Format
@@ -85,17 +85,17 @@ Padding shares vary based on namespace but they conform to the [share format](#s
 A namespace padding share uses the namespace of the blob that precedes it in the data square so that the data square can retain the property that all shares are ordered by namespace.
 A namespace padding share acts as padding between blobs so that the subsequent blob begins at an index that conforms to the [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules). Clients MAY ignore the contents of these shares because they don't contain any significant data.
 
-### Reserved Padding Share
+### Primary Reserved Padding Share
 
-Reserved padding shares use the [`RESERVED_PADDING_NAMESPACE`](./consensus.md#constants). Reserved padding shares are placed after the last reserved namespace share in the data square so that the first blob can start at an index that conforms to blob share commitment rules. Clients MAY ignore the contents of these shares because they don't contain any significant data.
+Primary reserved padding shares use the [`PRIMARY_RESERVED_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Primary reserved padding shares are placed after shares in the primary reserved namespace range so that the first blob can start at an index that conforms to blob share commitment rules. Clients MAY ignore the contents of these shares because they don't contain any significant data.
 
 ### Tail Padding Share
 
-Tail padding shares use the [`TAIL_PADDING_NAMESPACE`](./consensus.md#constants). Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients MAY ignore the contents of these shares because they don't contain any significant data.
+Tail padding shares use the [`TAIL_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients MAY ignore the contents of these shares because they don't contain any significant data.
 
 ## Parity Share
 
-Parity shares use the [`PARITY_SHARE_NAMESPACE`](./consensus.md#constants). Parity shares are the output of the erasure coding step of the data square construction process. They occupy quadrants Q1, Q2, and Q3 of the extended data square and are used to reconstruct the original data square (Q0). Bytes carry no special meaning.
+Parity shares use the [`PARITY_SHARE_NAMESPACE`](./namespace.md#reserved-namespaces). Parity shares are the output of the erasure coding step of the data square construction process. They occupy quadrants Q1, Q2, and Q3 of the extended data square and are used to reconstruct the original data square (Q0). Bytes carry no special meaning.
 
 ## Share Splitting