From 25b2df79c6d9ab47220be2b32f3daa5df9cc237e Mon Sep 17 00:00:00 2001 From: Rootul P Date: Mon, 14 Aug 2023 10:29:36 -0400 Subject: [PATCH 1/6] feat!: reserve the last 256 namespaces for protocol use (#2257) Closes https://github.com/celestiaorg/celestia-app/issues/2253 (cherry picked from commit cb8247ead807a52bb8915d450982544755efc1e6) # Conflicts: # pkg/namespace/consts.go # pkg/namespace/namespace.go # pkg/namespace/namespace_test.go # pkg/shares/padding.go # specs/src/specs/namespace.md --- pkg/namespace/consts.go | 52 +++++++++++++---- pkg/namespace/namespace.go | 16 +++++- pkg/namespace/namespace_test.go | 95 +++++++++++++++++++++++++++++++ pkg/shares/padding.go | 8 +++ pkg/shares/padding_test.go | 2 +- pkg/shares/parse_test.go | 4 +- pkg/shares/share_sequence_test.go | 2 +- pkg/shares/shares.go | 8 +-- specs/src/specs/namespace.md | 28 +++++++++ specs/src/specs/shares.md | 10 ++-- 10 files changed, 198 insertions(+), 27 deletions(-) diff --git a/pkg/namespace/consts.go b/pkg/namespace/consts.go index 2a71299592..d0ae48aec7 100644 --- a/pkg/namespace/consts.go +++ b/pkg/namespace/consts.go @@ -36,35 +36,63 @@ var ( // NamespaceVersionZeroPrefix is the prefix of a namespace ID for version 0. NamespaceVersionZeroPrefix = bytes.Repeat([]byte{0}, NamespaceVersionZeroPrefixSize) - // TxNamespace is the namespace reserved for transaction data. - TxNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 1}) + // TxNamespace is the namespace reserved for ordinary Cosmos SDK transactions. + TxNamespace = primaryReservedNamespace(0x01) // IntermediateStateRootsNamespace is the namespace reserved for // intermediate state root data. - IntermediateStateRootsNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 2}) + IntermediateStateRootsNamespace = primaryReservedNamespace(0x02) // PayForBlobNamespace is the namespace reserved for PayForBlobs transactions. - PayForBlobNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 4}) + PayForBlobNamespace = primaryReservedNamespace(0x04) - // ReservedPaddingNamespace is the namespace used for padding after all - // reserved namespaces. In practice this padding is after transactions - // (ordinary and PFBs) but before blobs. - ReservedPaddingNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) + // PrimaryReservedPaddingNamespace is the namespace used for padding after all + // primary reserved namespaces. + PrimaryReservedPaddingNamespace = primaryReservedNamespace(0xFF) +<<<<<<< HEAD // MaxReservedNamespace is lexicographically the largest namespace that is // reserved for protocol use. MaxReservedNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) +======= + // MaxPrimaryReservedNamespace is the highest primary reserved namespace. + // Namespaces lower than this are reserved for protocol use. + MaxPrimaryReservedNamespace = primaryReservedNamespace(0xFF) + + // MinSecondaryReservedNamespace is the lowest secondary reserved namespace + // reserved for protocol use. Namespaces higher than this are reserved for + // protocol use. + MinSecondaryReservedNamespace = secondaryReservedNamespace(0x00) +>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) // TailPaddingNamespace is the namespace reserved for tail padding. All data // with this namespace will be ignored. - TailPaddingNamespace = Namespace{ - Version: math.MaxUint8, - ID: append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), 0xFE), - } + TailPaddingNamespace = secondaryReservedNamespace(0xFE) // ParitySharesNamespace is the namespace reserved for erasure coded data. +<<<<<<< HEAD ParitySharesNamespace = Namespace{ Version: math.MaxUint8, ID: bytes.Repeat([]byte{0xFF}, NamespaceIDSize), } +======= + ParitySharesNamespace = secondaryReservedNamespace(0xFF) + + // SupportedBlobNamespaceVersions is a list of namespace versions that can be specified by a user for blobs. + SupportedBlobNamespaceVersions = []uint8{NamespaceVersionZero} +>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) ) + +func primaryReservedNamespace(lastByte byte) Namespace { + return Namespace{ + Version: NamespaceVersionZero, + ID: append(bytes.Repeat([]byte{0x00}, NamespaceIDSize-1), lastByte), + } +} + +func secondaryReservedNamespace(lastByte byte) Namespace { + return Namespace{ + Version: NamespaceVersionMax, + ID: append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), lastByte), + } +} diff --git a/pkg/namespace/namespace.go b/pkg/namespace/namespace.go index 5da38bc398..4eee45b1b3 100644 --- a/pkg/namespace/namespace.go +++ b/pkg/namespace/namespace.go @@ -122,7 +122,19 @@ func validateID(version uint8, id []byte) error { } func (n Namespace) IsReserved() bool { +<<<<<<< HEAD return bytes.Compare(n.Bytes(), MaxReservedNamespace.Bytes()) < 1 +======= + return n.IsPrimaryReserved() || n.IsSecondaryReserved() +} + +func (n Namespace) IsPrimaryReserved() bool { + return n.IsLessOrEqualThan(MaxPrimaryReservedNamespace) +} + +func (n Namespace) IsSecondaryReserved() bool { + return n.IsGreaterOrEqualThan(MinSecondaryReservedNamespace) +>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) } func (n Namespace) IsParityShares() bool { @@ -133,8 +145,8 @@ func (n Namespace) IsTailPadding() bool { return bytes.Equal(n.Bytes(), TailPaddingNamespace.Bytes()) } -func (n Namespace) IsReservedPadding() bool { - return bytes.Equal(n.Bytes(), ReservedPaddingNamespace.Bytes()) +func (n Namespace) IsPrimaryReservedPadding() bool { + return bytes.Equal(n.Bytes(), PrimaryReservedPaddingNamespace.Bytes()) } func (n Namespace) IsTx() bool { diff --git a/pkg/namespace/namespace_test.go b/pkg/namespace/namespace_test.go index f7b97487dc..ff36a37e07 100644 --- a/pkg/namespace/namespace_test.go +++ b/pkg/namespace/namespace_test.go @@ -2,6 +2,11 @@ package namespace import ( "bytes" +<<<<<<< HEAD +======= + "math" + "reflect" +>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) "testing" "github.com/stretchr/testify/assert" @@ -195,3 +200,93 @@ func TestBytes(t *testing.T) { assert.Equal(t, want, got) } +<<<<<<< HEAD +======= + +func TestLeftPad(t *testing.T) { + tests := []struct { + input []byte + size int + expected []byte + }{ + // input smaller than pad size + {[]byte{1, 2, 3}, 10, []byte{0, 0, 0, 0, 0, 0, 0, 1, 2, 3}}, + {[]byte{1}, 5, []byte{0, 0, 0, 0, 1}}, + {[]byte{1, 2}, 4, []byte{0, 0, 1, 2}}, + + // input equal to pad size + {[]byte{1, 2, 3}, 3, []byte{1, 2, 3}}, + {[]byte{1, 2, 3, 4}, 4, []byte{1, 2, 3, 4}}, + + // input larger than pad size + {[]byte{1, 2, 3, 4, 5}, 4, []byte{1, 2, 3, 4, 5}}, + {[]byte{1, 2, 3, 4, 5, 6, 7}, 3, []byte{1, 2, 3, 4, 5, 6, 7}}, + + // input size 0 + {[]byte{}, 8, []byte{0, 0, 0, 0, 0, 0, 0, 0}}, + {[]byte{}, 0, []byte{}}, + } + + for _, test := range tests { + result := leftPad(test.input, test.size) + assert.True(t, reflect.DeepEqual(result, test.expected)) + } +} + +func TestIsReserved(t *testing.T) { + type testCase struct { + ns Namespace + want bool + } + testCases := []testCase{ + { + ns: MustNewV0(bytes.Repeat([]byte{1}, NamespaceVersionZeroIDSize)), + want: false, + }, + { + ns: TxNamespace, + want: true, + }, + { + ns: IntermediateStateRootsNamespace, + want: true, + }, + { + ns: PayForBlobNamespace, + want: true, + }, + { + ns: PrimaryReservedPaddingNamespace, + want: true, + }, + { + ns: MaxPrimaryReservedNamespace, + want: true, + }, + { + ns: MinSecondaryReservedNamespace, + want: true, + }, + { + ns: TailPaddingNamespace, + want: true, + }, + { + ns: ParitySharesNamespace, + want: true, + }, + { + ns: Namespace{ + Version: math.MaxUint8, + ID: append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), 1), + }, + want: true, + }, + } + + for _, tc := range testCases { + got := tc.ns.IsReserved() + assert.Equal(t, tc.want, got) + } +} +>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) diff --git a/pkg/shares/padding.go b/pkg/shares/padding.go index d2ee22bb73..25fc91bedf 100644 --- a/pkg/shares/padding.go +++ b/pkg/shares/padding.go @@ -52,7 +52,11 @@ func NamespacePaddingShares(ns appns.Namespace, n int) ([]Share, error) { // first blob can start at an index that conforms to non-interactive default // rules. func ReservedPaddingShare() Share { +<<<<<<< HEAD share, err := NamespacePaddingShare(appns.ReservedPaddingNamespace) +======= + share, err := NamespacePaddingShare(appns.PrimaryReservedPaddingNamespace, appconsts.ShareVersionZero) +>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) if err != nil { panic(err) } @@ -61,7 +65,11 @@ func ReservedPaddingShare() Share { // ReservedPaddingShare returns n reserved padding shares. func ReservedPaddingShares(n int) []Share { +<<<<<<< HEAD shares, err := NamespacePaddingShares(appns.ReservedPaddingNamespace, n) +======= + shares, err := NamespacePaddingShares(appns.PrimaryReservedPaddingNamespace, appconsts.ShareVersionZero, n) +>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) if err != nil { panic(err) } diff --git a/pkg/shares/padding_test.go b/pkg/shares/padding_test.go index abf04e8515..1830ddcf0a 100644 --- a/pkg/shares/padding_test.go +++ b/pkg/shares/padding_test.go @@ -23,7 +23,7 @@ var nsOnePadding, _ = zeroPadIfNecessary( var reservedPadding, _ = zeroPadIfNecessary( append( - appns.ReservedPaddingNamespace.Bytes(), + appns.PrimaryReservedPaddingNamespace.Bytes(), []byte{ 1, // info byte 0, 0, 0, 0, // sequence len diff --git a/pkg/shares/parse_test.go b/pkg/shares/parse_test.go index 402b8714b7..3aba052739 100644 --- a/pkg/shares/parse_test.go +++ b/pkg/shares/parse_test.go @@ -162,11 +162,11 @@ func TestParseShares(t *testing.T) { ignorePadding: false, want: []ShareSequence{ { - Namespace: appns.ReservedPaddingNamespace, + Namespace: appns.PrimaryReservedPaddingNamespace, Shares: []Share{ReservedPaddingShare()}, }, { - Namespace: appns.ReservedPaddingNamespace, + Namespace: appns.PrimaryReservedPaddingNamespace, Shares: []Share{ReservedPaddingShare()}, }, }, diff --git a/pkg/shares/share_sequence_test.go b/pkg/shares/share_sequence_test.go index 31d790cdb4..9ad10376bb 100644 --- a/pkg/shares/share_sequence_test.go +++ b/pkg/shares/share_sequence_test.go @@ -159,7 +159,7 @@ func Test_validSequenceLen(t *testing.T) { } reservedPadding := ShareSequence{ - Namespace: appns.ReservedPaddingNamespace, + Namespace: appns.PrimaryReservedPaddingNamespace, Shares: []Share{ReservedPaddingShare()}, } diff --git a/pkg/shares/shares.go b/pkg/shares/shares.go index 1f805d4e17..93711cc639 100644 --- a/pkg/shares/shares.go +++ b/pkg/shares/shares.go @@ -121,11 +121,11 @@ func (s *Share) IsPadding() (bool, error) { if err != nil { return false, err } - isReservedPadding, err := s.isReservedPadding() + isPrimaryReservedPadding, err := s.isPrimaryReservedPadding() if err != nil { return false, err } - return isNamespacePadding || isTailPadding || isReservedPadding, nil + return isNamespacePadding || isTailPadding || isPrimaryReservedPadding, nil } func (s *Share) isNamespacePadding() (bool, error) { @@ -149,12 +149,12 @@ func (s *Share) isTailPadding() (bool, error) { return ns.IsTailPadding(), nil } -func (s *Share) isReservedPadding() (bool, error) { +func (s *Share) isPrimaryReservedPadding() (bool, error) { ns, err := s.Namespace() if err != nil { return false, err } - return ns.IsReservedPadding(), nil + return ns.IsPrimaryReservedPadding(), nil } func (s *Share) ToBytes() []byte { diff --git a/specs/src/specs/namespace.md b/specs/src/specs/namespace.md index b259516cdb..3fd9436b9a 100644 --- a/specs/src/specs/namespace.md +++ b/specs/src/specs/namespace.md @@ -42,6 +42,7 @@ The namespace ID is a 28 byte identifier that uniquely identifies a namespace. T ## Reserved Namespaces +<<<<<<< HEAD | name | type | value | description | |-------------------------------------|-------------|----------------------------------------------------------------|------------------------------------------------------------------------------------------------------| | `TRANSACTION_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000001` | Transactions: requests that modify the state. | @@ -51,6 +52,33 @@ The namespace ID is a 28 byte identifier that uniquely identifies a namespace. T | `MAX_RESERVED_NAMESPACE` | `Namespace` | `0x00000000000000000000000000000000000000000000000000000000FF` | Max reserved namespace is lexicographically the largest namespace that is reserved for protocol use. | | `TAIL_PADDING_NAMESPACE` | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Tail padding for blobs: padding after all blobs to fill up the original data square. | | `PARITY_SHARE_NAMESPACE` | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Parity shares: extended shares in the available data matrix. | +======= +Celestia reserves some namespaces for protocol use. +These namespaces are called "reserved namespaces". +Reserved namespaces are used to arrange the contents of the [data square](./data_square_layout.md). +Applications MUST NOT use reserved namespaces for their blob data. +Reserved namespaces fall into two categories: _Primary_ and _Secondary_. + +- Primary: Namespaces with values less than or equal to `0x00000000000000000000000000000000000000000000000000000000FF`. Primary namespaces always have a version of `0`. +- Secondary: Namespaces with values greater than or equal to `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00`. +Secondary namespaces always have a version of `255` (`0xFF`) so that they are placed after all user specifiable namespaces in a sorted data square. +The `PARITY_SHARE_NAMESPACE` uses version `255` (`0xFF`) to enable more efficient proof generation within the context of [nmt](https://github.com/celestiaorg/nmt), where it is used in conjunction with the `IgnoreMaxNamespace` feature. +The `TAIL_PADDING_NAMESPACE` uses the version `255` to ensure that padding shares are always placed at the end of the Celestia data square even if a new user-specifiable version is introduced. + +Below is a list of the current reserved namespaces. +For additional information on the significance and application of the reserved namespaces, please refer to the [Data Square Layout](./data_square_layout.md) specifications. + +| name | type | category | value | description | +|--------------------------------------|-------------|-----------|----------------------------------------------------------------|----------------------------------------------------------------------------| +| `TRANSACTION_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000001` | Namespace for ordinary Cosmos SDK transactions. | +| `INTERMEDIATE_STATE_ROOT_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000002` | Namespace for intermediate state roots (not currently utilized). | +| `PAY_FOR_BLOB_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000004` | Namespace for transactions that contain a PayForBlob. | +| `PRIMARY_RESERVED_PADDING_NAMESPACE` | `Namespace` | Primary | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for padding after all primary reserved namespaces. | +| `MAX_PRIMARY_RESERVED_NAMESPACE` | `Namespace` | Primary | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for the highest primary reserved namespace. | +| `MIN_SECONDARY_RESERVED_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00` | Namespace for the lowest secondary reserved namespace. | +| `TAIL_PADDING_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Namespace for padding after all blobs to fill up the original data square. | +| `PARITY_SHARE_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Namespace for parity shares. | +>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) ## Assumptions and Considerations diff --git a/specs/src/specs/shares.md b/specs/src/specs/shares.md index 1c9fcfb2e4..cfa57e2f25 100644 --- a/specs/src/specs/shares.md +++ b/specs/src/specs/shares.md @@ -18,7 +18,7 @@ User submitted transactions are split into shares (see [share splitting](#share- [Padding](#padding) shares are added to the `k * k` matrix to ensure: -1. Blob sequences start on an index that conforms to [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) (see [namespace padding share](#namespace-padding-share) and [reserved padding share](#reserved-padding-share)) +1. Blob sequences start on an index that conforms to [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) (see [namespace padding share](#namespace-padding-share) and [reserved padding share](#primary-reserved-padding-share)) 1. The number of shares in the matrix is a perfect square (see [tail padding share](#tail-padding-share)) ## Share Format @@ -85,17 +85,17 @@ Padding shares vary based on namespace but they conform to the [share format](#s A namespace padding share uses the namespace of the blob that precedes it in the data square so that the data square can retain the property that all shares are ordered by namespace. A namespace padding share acts as padding between blobs so that the subsequent blob begins at an index that conforms to the [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules). Clients MAY ignore the contents of these shares because they don't contain any significant data. -### Reserved Padding Share +### Primary Reserved Padding Share -Reserved padding shares use the [`RESERVED_PADDING_NAMESPACE`](./consensus.md#constants). Reserved padding shares are placed after the last reserved namespace share in the data square so that the first blob can start at an index that conforms to blob share commitment rules. Clients MAY ignore the contents of these shares because they don't contain any significant data. +Primary reserved padding shares use the [`PRIMARY_RESERVED_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Primary reserved padding shares are placed after shares in the primary reserved namespace range so that the first blob can start at an index that conforms to blob share commitment rules. Clients MAY ignore the contents of these shares because they don't contain any significant data. ### Tail Padding Share -Tail padding shares use the [`TAIL_PADDING_NAMESPACE`](./consensus.md#constants). Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients MAY ignore the contents of these shares because they don't contain any significant data. +Tail padding shares use the [`TAIL_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients MAY ignore the contents of these shares because they don't contain any significant data. ## Parity Share -Parity shares use the [`PARITY_SHARE_NAMESPACE`](./consensus.md#constants). Parity shares are the output of the erasure coding step of the data square construction process. They occupy quadrants Q1, Q2, and Q3 of the extended data square and are used to reconstruct the original data square (Q0). Bytes carry no special meaning. +Parity shares use the [`PARITY_SHARE_NAMESPACE`](./namespace.md#reserved-namespaces). Parity shares are the output of the erasure coding step of the data square construction process. They occupy quadrants Q1, Q2, and Q3 of the extended data square and are used to reconstruct the original data square (Q0). Bytes carry no special meaning. ## Share Splitting From b76d74c7069bb9f9632b20a0f00fd09cbe88eb25 Mon Sep 17 00:00:00 2001 From: Rootul Patel Date: Mon, 14 Aug 2023 15:04:36 -0400 Subject: [PATCH 2/6] Revert "feat!: reserve the last 256 namespaces for protocol use (#2257)" This reverts commit 25b2df79c6d9ab47220be2b32f3daa5df9cc237e. --- pkg/namespace/consts.go | 52 ++++------------- pkg/namespace/namespace.go | 16 +----- pkg/namespace/namespace_test.go | 95 ------------------------------- pkg/shares/padding.go | 8 --- pkg/shares/padding_test.go | 2 +- pkg/shares/parse_test.go | 4 +- pkg/shares/share_sequence_test.go | 2 +- pkg/shares/shares.go | 8 +-- specs/src/specs/namespace.md | 28 --------- specs/src/specs/shares.md | 10 ++-- 10 files changed, 27 insertions(+), 198 deletions(-) diff --git a/pkg/namespace/consts.go b/pkg/namespace/consts.go index d0ae48aec7..2a71299592 100644 --- a/pkg/namespace/consts.go +++ b/pkg/namespace/consts.go @@ -36,63 +36,35 @@ var ( // NamespaceVersionZeroPrefix is the prefix of a namespace ID for version 0. NamespaceVersionZeroPrefix = bytes.Repeat([]byte{0}, NamespaceVersionZeroPrefixSize) - // TxNamespace is the namespace reserved for ordinary Cosmos SDK transactions. - TxNamespace = primaryReservedNamespace(0x01) + // TxNamespace is the namespace reserved for transaction data. + TxNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 1}) // IntermediateStateRootsNamespace is the namespace reserved for // intermediate state root data. - IntermediateStateRootsNamespace = primaryReservedNamespace(0x02) + IntermediateStateRootsNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 2}) // PayForBlobNamespace is the namespace reserved for PayForBlobs transactions. - PayForBlobNamespace = primaryReservedNamespace(0x04) + PayForBlobNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 4}) - // PrimaryReservedPaddingNamespace is the namespace used for padding after all - // primary reserved namespaces. - PrimaryReservedPaddingNamespace = primaryReservedNamespace(0xFF) + // ReservedPaddingNamespace is the namespace used for padding after all + // reserved namespaces. In practice this padding is after transactions + // (ordinary and PFBs) but before blobs. + ReservedPaddingNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) -<<<<<<< HEAD // MaxReservedNamespace is lexicographically the largest namespace that is // reserved for protocol use. MaxReservedNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) -======= - // MaxPrimaryReservedNamespace is the highest primary reserved namespace. - // Namespaces lower than this are reserved for protocol use. - MaxPrimaryReservedNamespace = primaryReservedNamespace(0xFF) - - // MinSecondaryReservedNamespace is the lowest secondary reserved namespace - // reserved for protocol use. Namespaces higher than this are reserved for - // protocol use. - MinSecondaryReservedNamespace = secondaryReservedNamespace(0x00) ->>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) // TailPaddingNamespace is the namespace reserved for tail padding. All data // with this namespace will be ignored. - TailPaddingNamespace = secondaryReservedNamespace(0xFE) + TailPaddingNamespace = Namespace{ + Version: math.MaxUint8, + ID: append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), 0xFE), + } // ParitySharesNamespace is the namespace reserved for erasure coded data. -<<<<<<< HEAD ParitySharesNamespace = Namespace{ Version: math.MaxUint8, ID: bytes.Repeat([]byte{0xFF}, NamespaceIDSize), } -======= - ParitySharesNamespace = secondaryReservedNamespace(0xFF) - - // SupportedBlobNamespaceVersions is a list of namespace versions that can be specified by a user for blobs. - SupportedBlobNamespaceVersions = []uint8{NamespaceVersionZero} ->>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) ) - -func primaryReservedNamespace(lastByte byte) Namespace { - return Namespace{ - Version: NamespaceVersionZero, - ID: append(bytes.Repeat([]byte{0x00}, NamespaceIDSize-1), lastByte), - } -} - -func secondaryReservedNamespace(lastByte byte) Namespace { - return Namespace{ - Version: NamespaceVersionMax, - ID: append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), lastByte), - } -} diff --git a/pkg/namespace/namespace.go b/pkg/namespace/namespace.go index 4eee45b1b3..5da38bc398 100644 --- a/pkg/namespace/namespace.go +++ b/pkg/namespace/namespace.go @@ -122,19 +122,7 @@ func validateID(version uint8, id []byte) error { } func (n Namespace) IsReserved() bool { -<<<<<<< HEAD return bytes.Compare(n.Bytes(), MaxReservedNamespace.Bytes()) < 1 -======= - return n.IsPrimaryReserved() || n.IsSecondaryReserved() -} - -func (n Namespace) IsPrimaryReserved() bool { - return n.IsLessOrEqualThan(MaxPrimaryReservedNamespace) -} - -func (n Namespace) IsSecondaryReserved() bool { - return n.IsGreaterOrEqualThan(MinSecondaryReservedNamespace) ->>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) } func (n Namespace) IsParityShares() bool { @@ -145,8 +133,8 @@ func (n Namespace) IsTailPadding() bool { return bytes.Equal(n.Bytes(), TailPaddingNamespace.Bytes()) } -func (n Namespace) IsPrimaryReservedPadding() bool { - return bytes.Equal(n.Bytes(), PrimaryReservedPaddingNamespace.Bytes()) +func (n Namespace) IsReservedPadding() bool { + return bytes.Equal(n.Bytes(), ReservedPaddingNamespace.Bytes()) } func (n Namespace) IsTx() bool { diff --git a/pkg/namespace/namespace_test.go b/pkg/namespace/namespace_test.go index ff36a37e07..f7b97487dc 100644 --- a/pkg/namespace/namespace_test.go +++ b/pkg/namespace/namespace_test.go @@ -2,11 +2,6 @@ package namespace import ( "bytes" -<<<<<<< HEAD -======= - "math" - "reflect" ->>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) "testing" "github.com/stretchr/testify/assert" @@ -200,93 +195,3 @@ func TestBytes(t *testing.T) { assert.Equal(t, want, got) } -<<<<<<< HEAD -======= - -func TestLeftPad(t *testing.T) { - tests := []struct { - input []byte - size int - expected []byte - }{ - // input smaller than pad size - {[]byte{1, 2, 3}, 10, []byte{0, 0, 0, 0, 0, 0, 0, 1, 2, 3}}, - {[]byte{1}, 5, []byte{0, 0, 0, 0, 1}}, - {[]byte{1, 2}, 4, []byte{0, 0, 1, 2}}, - - // input equal to pad size - {[]byte{1, 2, 3}, 3, []byte{1, 2, 3}}, - {[]byte{1, 2, 3, 4}, 4, []byte{1, 2, 3, 4}}, - - // input larger than pad size - {[]byte{1, 2, 3, 4, 5}, 4, []byte{1, 2, 3, 4, 5}}, - {[]byte{1, 2, 3, 4, 5, 6, 7}, 3, []byte{1, 2, 3, 4, 5, 6, 7}}, - - // input size 0 - {[]byte{}, 8, []byte{0, 0, 0, 0, 0, 0, 0, 0}}, - {[]byte{}, 0, []byte{}}, - } - - for _, test := range tests { - result := leftPad(test.input, test.size) - assert.True(t, reflect.DeepEqual(result, test.expected)) - } -} - -func TestIsReserved(t *testing.T) { - type testCase struct { - ns Namespace - want bool - } - testCases := []testCase{ - { - ns: MustNewV0(bytes.Repeat([]byte{1}, NamespaceVersionZeroIDSize)), - want: false, - }, - { - ns: TxNamespace, - want: true, - }, - { - ns: IntermediateStateRootsNamespace, - want: true, - }, - { - ns: PayForBlobNamespace, - want: true, - }, - { - ns: PrimaryReservedPaddingNamespace, - want: true, - }, - { - ns: MaxPrimaryReservedNamespace, - want: true, - }, - { - ns: MinSecondaryReservedNamespace, - want: true, - }, - { - ns: TailPaddingNamespace, - want: true, - }, - { - ns: ParitySharesNamespace, - want: true, - }, - { - ns: Namespace{ - Version: math.MaxUint8, - ID: append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), 1), - }, - want: true, - }, - } - - for _, tc := range testCases { - got := tc.ns.IsReserved() - assert.Equal(t, tc.want, got) - } -} ->>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) diff --git a/pkg/shares/padding.go b/pkg/shares/padding.go index 25fc91bedf..d2ee22bb73 100644 --- a/pkg/shares/padding.go +++ b/pkg/shares/padding.go @@ -52,11 +52,7 @@ func NamespacePaddingShares(ns appns.Namespace, n int) ([]Share, error) { // first blob can start at an index that conforms to non-interactive default // rules. func ReservedPaddingShare() Share { -<<<<<<< HEAD share, err := NamespacePaddingShare(appns.ReservedPaddingNamespace) -======= - share, err := NamespacePaddingShare(appns.PrimaryReservedPaddingNamespace, appconsts.ShareVersionZero) ->>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) if err != nil { panic(err) } @@ -65,11 +61,7 @@ func ReservedPaddingShare() Share { // ReservedPaddingShare returns n reserved padding shares. func ReservedPaddingShares(n int) []Share { -<<<<<<< HEAD shares, err := NamespacePaddingShares(appns.ReservedPaddingNamespace, n) -======= - shares, err := NamespacePaddingShares(appns.PrimaryReservedPaddingNamespace, appconsts.ShareVersionZero, n) ->>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) if err != nil { panic(err) } diff --git a/pkg/shares/padding_test.go b/pkg/shares/padding_test.go index 1830ddcf0a..abf04e8515 100644 --- a/pkg/shares/padding_test.go +++ b/pkg/shares/padding_test.go @@ -23,7 +23,7 @@ var nsOnePadding, _ = zeroPadIfNecessary( var reservedPadding, _ = zeroPadIfNecessary( append( - appns.PrimaryReservedPaddingNamespace.Bytes(), + appns.ReservedPaddingNamespace.Bytes(), []byte{ 1, // info byte 0, 0, 0, 0, // sequence len diff --git a/pkg/shares/parse_test.go b/pkg/shares/parse_test.go index 3aba052739..402b8714b7 100644 --- a/pkg/shares/parse_test.go +++ b/pkg/shares/parse_test.go @@ -162,11 +162,11 @@ func TestParseShares(t *testing.T) { ignorePadding: false, want: []ShareSequence{ { - Namespace: appns.PrimaryReservedPaddingNamespace, + Namespace: appns.ReservedPaddingNamespace, Shares: []Share{ReservedPaddingShare()}, }, { - Namespace: appns.PrimaryReservedPaddingNamespace, + Namespace: appns.ReservedPaddingNamespace, Shares: []Share{ReservedPaddingShare()}, }, }, diff --git a/pkg/shares/share_sequence_test.go b/pkg/shares/share_sequence_test.go index 9ad10376bb..31d790cdb4 100644 --- a/pkg/shares/share_sequence_test.go +++ b/pkg/shares/share_sequence_test.go @@ -159,7 +159,7 @@ func Test_validSequenceLen(t *testing.T) { } reservedPadding := ShareSequence{ - Namespace: appns.PrimaryReservedPaddingNamespace, + Namespace: appns.ReservedPaddingNamespace, Shares: []Share{ReservedPaddingShare()}, } diff --git a/pkg/shares/shares.go b/pkg/shares/shares.go index 93711cc639..1f805d4e17 100644 --- a/pkg/shares/shares.go +++ b/pkg/shares/shares.go @@ -121,11 +121,11 @@ func (s *Share) IsPadding() (bool, error) { if err != nil { return false, err } - isPrimaryReservedPadding, err := s.isPrimaryReservedPadding() + isReservedPadding, err := s.isReservedPadding() if err != nil { return false, err } - return isNamespacePadding || isTailPadding || isPrimaryReservedPadding, nil + return isNamespacePadding || isTailPadding || isReservedPadding, nil } func (s *Share) isNamespacePadding() (bool, error) { @@ -149,12 +149,12 @@ func (s *Share) isTailPadding() (bool, error) { return ns.IsTailPadding(), nil } -func (s *Share) isPrimaryReservedPadding() (bool, error) { +func (s *Share) isReservedPadding() (bool, error) { ns, err := s.Namespace() if err != nil { return false, err } - return ns.IsPrimaryReservedPadding(), nil + return ns.IsReservedPadding(), nil } func (s *Share) ToBytes() []byte { diff --git a/specs/src/specs/namespace.md b/specs/src/specs/namespace.md index 3fd9436b9a..b259516cdb 100644 --- a/specs/src/specs/namespace.md +++ b/specs/src/specs/namespace.md @@ -42,7 +42,6 @@ The namespace ID is a 28 byte identifier that uniquely identifies a namespace. T ## Reserved Namespaces -<<<<<<< HEAD | name | type | value | description | |-------------------------------------|-------------|----------------------------------------------------------------|------------------------------------------------------------------------------------------------------| | `TRANSACTION_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000001` | Transactions: requests that modify the state. | @@ -52,33 +51,6 @@ The namespace ID is a 28 byte identifier that uniquely identifies a namespace. T | `MAX_RESERVED_NAMESPACE` | `Namespace` | `0x00000000000000000000000000000000000000000000000000000000FF` | Max reserved namespace is lexicographically the largest namespace that is reserved for protocol use. | | `TAIL_PADDING_NAMESPACE` | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Tail padding for blobs: padding after all blobs to fill up the original data square. | | `PARITY_SHARE_NAMESPACE` | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Parity shares: extended shares in the available data matrix. | -======= -Celestia reserves some namespaces for protocol use. -These namespaces are called "reserved namespaces". -Reserved namespaces are used to arrange the contents of the [data square](./data_square_layout.md). -Applications MUST NOT use reserved namespaces for their blob data. -Reserved namespaces fall into two categories: _Primary_ and _Secondary_. - -- Primary: Namespaces with values less than or equal to `0x00000000000000000000000000000000000000000000000000000000FF`. Primary namespaces always have a version of `0`. -- Secondary: Namespaces with values greater than or equal to `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00`. -Secondary namespaces always have a version of `255` (`0xFF`) so that they are placed after all user specifiable namespaces in a sorted data square. -The `PARITY_SHARE_NAMESPACE` uses version `255` (`0xFF`) to enable more efficient proof generation within the context of [nmt](https://github.com/celestiaorg/nmt), where it is used in conjunction with the `IgnoreMaxNamespace` feature. -The `TAIL_PADDING_NAMESPACE` uses the version `255` to ensure that padding shares are always placed at the end of the Celestia data square even if a new user-specifiable version is introduced. - -Below is a list of the current reserved namespaces. -For additional information on the significance and application of the reserved namespaces, please refer to the [Data Square Layout](./data_square_layout.md) specifications. - -| name | type | category | value | description | -|--------------------------------------|-------------|-----------|----------------------------------------------------------------|----------------------------------------------------------------------------| -| `TRANSACTION_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000001` | Namespace for ordinary Cosmos SDK transactions. | -| `INTERMEDIATE_STATE_ROOT_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000002` | Namespace for intermediate state roots (not currently utilized). | -| `PAY_FOR_BLOB_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000004` | Namespace for transactions that contain a PayForBlob. | -| `PRIMARY_RESERVED_PADDING_NAMESPACE` | `Namespace` | Primary | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for padding after all primary reserved namespaces. | -| `MAX_PRIMARY_RESERVED_NAMESPACE` | `Namespace` | Primary | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for the highest primary reserved namespace. | -| `MIN_SECONDARY_RESERVED_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00` | Namespace for the lowest secondary reserved namespace. | -| `TAIL_PADDING_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Namespace for padding after all blobs to fill up the original data square. | -| `PARITY_SHARE_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Namespace for parity shares. | ->>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) ## Assumptions and Considerations diff --git a/specs/src/specs/shares.md b/specs/src/specs/shares.md index cfa57e2f25..1c9fcfb2e4 100644 --- a/specs/src/specs/shares.md +++ b/specs/src/specs/shares.md @@ -18,7 +18,7 @@ User submitted transactions are split into shares (see [share splitting](#share- [Padding](#padding) shares are added to the `k * k` matrix to ensure: -1. Blob sequences start on an index that conforms to [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) (see [namespace padding share](#namespace-padding-share) and [reserved padding share](#primary-reserved-padding-share)) +1. Blob sequences start on an index that conforms to [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) (see [namespace padding share](#namespace-padding-share) and [reserved padding share](#reserved-padding-share)) 1. The number of shares in the matrix is a perfect square (see [tail padding share](#tail-padding-share)) ## Share Format @@ -85,17 +85,17 @@ Padding shares vary based on namespace but they conform to the [share format](#s A namespace padding share uses the namespace of the blob that precedes it in the data square so that the data square can retain the property that all shares are ordered by namespace. A namespace padding share acts as padding between blobs so that the subsequent blob begins at an index that conforms to the [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules). Clients MAY ignore the contents of these shares because they don't contain any significant data. -### Primary Reserved Padding Share +### Reserved Padding Share -Primary reserved padding shares use the [`PRIMARY_RESERVED_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Primary reserved padding shares are placed after shares in the primary reserved namespace range so that the first blob can start at an index that conforms to blob share commitment rules. Clients MAY ignore the contents of these shares because they don't contain any significant data. +Reserved padding shares use the [`RESERVED_PADDING_NAMESPACE`](./consensus.md#constants). Reserved padding shares are placed after the last reserved namespace share in the data square so that the first blob can start at an index that conforms to blob share commitment rules. Clients MAY ignore the contents of these shares because they don't contain any significant data. ### Tail Padding Share -Tail padding shares use the [`TAIL_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients MAY ignore the contents of these shares because they don't contain any significant data. +Tail padding shares use the [`TAIL_PADDING_NAMESPACE`](./consensus.md#constants). Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients MAY ignore the contents of these shares because they don't contain any significant data. ## Parity Share -Parity shares use the [`PARITY_SHARE_NAMESPACE`](./namespace.md#reserved-namespaces). Parity shares are the output of the erasure coding step of the data square construction process. They occupy quadrants Q1, Q2, and Q3 of the extended data square and are used to reconstruct the original data square (Q0). Bytes carry no special meaning. +Parity shares use the [`PARITY_SHARE_NAMESPACE`](./consensus.md#constants). Parity shares are the output of the erasure coding step of the data square construction process. They occupy quadrants Q1, Q2, and Q3 of the extended data square and are used to reconstruct the original data square (Q0). Bytes carry no special meaning. ## Share Splitting From 6f8ece4f49c88737d4801543143b3268528ef1de Mon Sep 17 00:00:00 2001 From: Rootul P Date: Thu, 27 Jul 2023 08:12:25 -0400 Subject: [PATCH 3/6] refactor!: un-export `namespace.ValidateBlobNamespace` (#2145) Closes https://github.com/celestiaorg/celestia-app/issues/2143 De-duplicate `ValidateBlobNamespace` and export it from `x/blob/types` package. The namespace package needs to define its own `isBlobNamespace` because it can't import `ValidateBlobNamespace` from `x/blob/types` as that would introduce a circular dependency. --- pkg/namespace/consts.go | 3 +++ pkg/namespace/namespace.go | 23 +++-------------------- pkg/namespace/random_blob.go | 29 +++++++++++++++++++++++++---- x/blob/types/blob_tx.go | 2 +- x/blob/types/payforblob.go | 9 +++++---- x/blob/types/payforblob_test.go | 28 ++++++++++++++-------------- 6 files changed, 51 insertions(+), 43 deletions(-) diff --git a/pkg/namespace/consts.go b/pkg/namespace/consts.go index 2a71299592..64b5f8db84 100644 --- a/pkg/namespace/consts.go +++ b/pkg/namespace/consts.go @@ -67,4 +67,7 @@ var ( Version: math.MaxUint8, ID: bytes.Repeat([]byte{0xFF}, NamespaceIDSize), } + + // SupportedBlobNamespaceVersions is a list of namespace versions that can be specified by a user for blobs. + SupportedBlobNamespaceVersions = []uint8{NamespaceVersionZero} ) diff --git a/pkg/namespace/namespace.go b/pkg/namespace/namespace.go index 5da38bc398..f6af33c57e 100644 --- a/pkg/namespace/namespace.go +++ b/pkg/namespace/namespace.go @@ -12,7 +12,7 @@ type Namespace struct { // New returns a new namespace with the provided version and id. func New(version uint8, id []byte) (Namespace, error) { - err := validateVersion(version) + err := validateVersionSupported(version) if err != nil { return Namespace{}, err } @@ -83,25 +83,8 @@ func (n Namespace) Bytes() []byte { return append([]byte{n.Version}, n.ID...) } -// ValidateBlobNamespace returns an error if this namespace is not a valid blob namespace. -func (n Namespace) ValidateBlobNamespace() error { - if n.IsReserved() { - return fmt.Errorf("invalid blob namespace: %v cannot use a reserved namespace ID, want > %v", n.Bytes(), MaxReservedNamespace.Bytes()) - } - - if n.IsParityShares() { - return fmt.Errorf("invalid blob namespace: %v cannot use parity shares namespace ID", n.Bytes()) - } - - if n.IsTailPadding() { - return fmt.Errorf("invalid blob namespace: %v cannot use tail padding namespace ID", n.Bytes()) - } - - return nil -} - -// validateVersion returns an error if the version is not supported. -func validateVersion(version uint8) error { +// validateVersionSupported returns an error if the version is not supported. +func validateVersionSupported(version uint8) error { if version != NamespaceVersionZero && version != NamespaceVersionMax { return fmt.Errorf("unsupported namespace version %v", version) } diff --git a/pkg/namespace/random_blob.go b/pkg/namespace/random_blob.go index 166f114acb..1bc459ca29 100644 --- a/pkg/namespace/random_blob.go +++ b/pkg/namespace/random_blob.go @@ -2,6 +2,7 @@ package namespace import ( tmrand "github.com/tendermint/tendermint/libs/rand" + "golang.org/x/exp/slices" ) func RandomBlobNamespaceID() []byte { @@ -22,11 +23,9 @@ func RandomBlobNamespaceWithPRG(prg *tmrand.Rand) Namespace { for { id := RandomBlobNamespaceIDWithPRG(prg) namespace := MustNewV0(id) - err := namespace.ValidateBlobNamespace() - if err != nil { - continue + if isBlobNamespace(namespace) { + return namespace } - return namespace } } @@ -36,3 +35,25 @@ func RandomBlobNamespaces(rand *tmrand.Rand, count int) (namespaces []Namespace) } return namespaces } + +// isBlobNamespace returns an true if this namespace is a valid user-specifiable +// blob namespace. +func isBlobNamespace(ns Namespace) bool { + if ns.IsReserved() { + return false + } + + if ns.IsParityShares() { + return false + } + + if ns.IsTailPadding() { + return false + } + + if !slices.Contains(SupportedBlobNamespaceVersions, ns.Version) { + return false + } + + return true +} diff --git a/x/blob/types/blob_tx.go b/x/blob/types/blob_tx.go index 5c9276a7ea..125c282315 100644 --- a/x/blob/types/blob_tx.go +++ b/x/blob/types/blob_tx.go @@ -20,7 +20,7 @@ type Blob = tmproto.Blob // NewBlob creates a new coretypes.Blob from the provided data after performing // basic stateless checks over it. func NewBlob(ns appns.Namespace, blob []byte, shareVersion uint8) (*Blob, error) { - err := ns.ValidateBlobNamespace() + err := ValidateBlobNamespace(ns) if err != nil { return nil, err } diff --git a/x/blob/types/payforblob.go b/x/blob/types/payforblob.go index 1685e03c31..00bd9e8b41 100644 --- a/x/blob/types/payforblob.go +++ b/x/blob/types/payforblob.go @@ -181,8 +181,9 @@ func DefaultEstimateGas(blobSizes []uint32) uint64 { return EstimateGas(blobSizes, appconsts.DefaultGasPerBlobByte, auth.DefaultTxSizeCostPerByte) } -// ValidateBlobNamespace returns an error if the provided namespace is reserved, -// parity shares, or tail padding. +// ValidateBlobNamespace returns an error if the provided namespace is an +// invalid user-specifiable blob namespace (e.g. reserved, parity shares, or +// tail padding). func ValidateBlobNamespace(ns appns.Namespace) error { if ns.IsReserved() { return ErrReservedNamespace.Wrapf("got namespace: %x, want: > %x", ns, appns.MaxReservedNamespace) @@ -196,7 +197,7 @@ func ValidateBlobNamespace(ns appns.Namespace) error { return ErrTailPaddingNamespace } - if ns.Version != appns.NamespaceVersionZero { + if !slices.Contains(appns.SupportedBlobNamespaceVersions, ns.Version) { return ErrInvalidNamespaceVersion } @@ -313,7 +314,7 @@ func ValidateBlobs(blobs ...*Blob) error { if err != nil { return err } - err = ns.ValidateBlobNamespace() + err = ValidateBlobNamespace(ns) if err != nil { return err } diff --git a/x/blob/types/payforblob_test.go b/x/blob/types/payforblob_test.go index 90a16a57d8..cfdaa1d844 100644 --- a/x/blob/types/payforblob_test.go +++ b/x/blob/types/payforblob_test.go @@ -123,23 +123,23 @@ func TestValidateBasic(t *testing.T) { validMsg := validMsgPayForBlobs(t) - // MsgPayForBlobs that uses parity shares namespace id + // MsgPayForBlobs that uses parity shares namespace paritySharesMsg := validMsgPayForBlobs(t) paritySharesMsg.Namespaces[0] = appns.ParitySharesNamespace.Bytes() - // MsgPayForBlobs that uses tail padding namespace id + // MsgPayForBlobs that uses tail padding namespace tailPaddingMsg := validMsgPayForBlobs(t) tailPaddingMsg.Namespaces[0] = appns.TailPaddingNamespace.Bytes() - // MsgPayForBlobs that uses transaction namespace id + // MsgPayForBlobs that uses transaction namespace txNamespaceMsg := validMsgPayForBlobs(t) txNamespaceMsg.Namespaces[0] = appns.TxNamespace.Bytes() - // MsgPayForBlobs that uses intermediateStateRoots namespace id + // MsgPayForBlobs that uses intermediateStateRoots namespace intermediateStateRootsNamespaceMsg := validMsgPayForBlobs(t) intermediateStateRootsNamespaceMsg.Namespaces[0] = appns.IntermediateStateRootsNamespace.Bytes() - // MsgPayForBlobs that uses the max reserved namespace id + // MsgPayForBlobs that uses the max reserved namespace maxReservedNamespaceMsg := validMsgPayForBlobs(t) maxReservedNamespaceMsg.Namespaces[0] = appns.MaxReservedNamespace.Bytes() @@ -151,9 +151,9 @@ func TestValidateBasic(t *testing.T) { invalidShareCommitmentSize := validMsgPayForBlobs(t) invalidShareCommitmentSize.ShareCommitments[0] = bytes.Repeat([]byte{0x1}, 31) - // MsgPayForBlobs that has no namespace ids - noNamespaceIds := validMsgPayForBlobs(t) - noNamespaceIds.Namespaces = [][]byte{} + // MsgPayForBlobs that has no namespaces + noNamespaces := validMsgPayForBlobs(t) + noNamespaces.Namespaces = [][]byte{} // MsgPayForBlobs that has no share versions noShareVersions := validMsgPayForBlobs(t) @@ -174,27 +174,27 @@ func TestValidateBasic(t *testing.T) { wantErr: nil, }, { - name: "parity shares namespace id", + name: "parity shares namespace", msg: paritySharesMsg, wantErr: ErrParitySharesNamespace, }, { - name: "tail padding namespace id", + name: "tail padding namespace", msg: tailPaddingMsg, wantErr: ErrTailPaddingNamespace, }, { - name: "transaction namspace namespace id", + name: "tx namespace", msg: txNamespaceMsg, wantErr: ErrReservedNamespace, }, { - name: "intermediate state root namespace id", + name: "intermediate state root namespace", msg: intermediateStateRootsNamespaceMsg, wantErr: ErrReservedNamespace, }, { - name: "max reserved namespace id", + name: "max reserved namespace", msg: maxReservedNamespaceMsg, wantErr: ErrReservedNamespace, }, @@ -210,7 +210,7 @@ func TestValidateBasic(t *testing.T) { }, { name: "no namespace ids", - msg: noNamespaceIds, + msg: noNamespaces, wantErr: ErrNoNamespaces, }, { From 5845ea423d36138cc44a1ea51ef5fa69314c5693 Mon Sep 17 00:00:00 2001 From: Sanaz Taheri <35961250+staheri14@users.noreply.github.com> Date: Thu, 3 Aug 2023 16:12:50 +0300 Subject: [PATCH 4/6] feat!: incorporates ParitySharesNamespace and TailPaddingNamespace into IsReserved function (#2194) Closes #https://github.com/celestiaorg/celestia-app/issues/2138 This PR also addresses the concerns raised in the [comment](https://github.com/celestiaorg/celestia-app/issues/2138#issuecomment-1657219076) by introducing a distinction between two types of reserved namespaces. Namespaces within the range [0-255] are now identified as "Primary Reserved Namespaces," while any other reserved namespaces beyond this range are categorized as "Non-Primary Reserved Namespaces." Hopefully, this differentiation provides better clarity and organization regarding namespace allocation and usage within the code. - [x] New and updated code has appropriate documentation - [x] New and updated code has new and/or updated testing - [x] Required CI checks are passing - [x] Visual proof for any user facing features like CLI or documentation updates - [x] Linked issues closed with keywords --- pkg/namespace/consts.go | 7 ++++--- pkg/namespace/namespace.go | 6 +++++- pkg/namespace/random_blob.go | 8 -------- specs/src/specs/namespace.md | 33 ++++++++++++++++++++++++--------- x/blob/types/payforblob.go | 10 +--------- x/blob/types/payforblob_test.go | 8 ++++---- 6 files changed, 38 insertions(+), 34 deletions(-) diff --git a/pkg/namespace/consts.go b/pkg/namespace/consts.go index 64b5f8db84..df30ba4e21 100644 --- a/pkg/namespace/consts.go +++ b/pkg/namespace/consts.go @@ -51,9 +51,10 @@ var ( // (ordinary and PFBs) but before blobs. ReservedPaddingNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) - // MaxReservedNamespace is lexicographically the largest namespace that is - // reserved for protocol use. - MaxReservedNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) + // MaxPrimaryReservedNamespace represents the largest primary reserved + // namespace reserved for protocol use. Note that there may be other + // non-primary reserved namespaces beyond this upper limit. + MaxPrimaryReservedNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) // TailPaddingNamespace is the namespace reserved for tail padding. All data // with this namespace will be ignored. diff --git a/pkg/namespace/namespace.go b/pkg/namespace/namespace.go index f6af33c57e..d3ea522480 100644 --- a/pkg/namespace/namespace.go +++ b/pkg/namespace/namespace.go @@ -104,8 +104,12 @@ func validateID(version uint8, id []byte) error { return nil } +// IsReserved returns true if the namespace is reserved for protocol-use. func (n Namespace) IsReserved() bool { - return bytes.Compare(n.Bytes(), MaxReservedNamespace.Bytes()) < 1 + isLessThanOrEqualToMaxPrimaryReservedNamespace := bytes.Compare(n.Bytes(), MaxPrimaryReservedNamespace.Bytes()) < 1 + isParityNamespace := n.IsParityShares() + isTailPadding := n.IsTailPadding() + return isLessThanOrEqualToMaxPrimaryReservedNamespace || isParityNamespace || isTailPadding } func (n Namespace) IsParityShares() bool { diff --git a/pkg/namespace/random_blob.go b/pkg/namespace/random_blob.go index 1bc459ca29..156d8a574f 100644 --- a/pkg/namespace/random_blob.go +++ b/pkg/namespace/random_blob.go @@ -43,14 +43,6 @@ func isBlobNamespace(ns Namespace) bool { return false } - if ns.IsParityShares() { - return false - } - - if ns.IsTailPadding() { - return false - } - if !slices.Contains(SupportedBlobNamespaceVersions, ns.Version) { return false } diff --git a/specs/src/specs/namespace.md b/specs/src/specs/namespace.md index b259516cdb..a56ac5a582 100644 --- a/specs/src/specs/namespace.md +++ b/specs/src/specs/namespace.md @@ -42,15 +42,30 @@ The namespace ID is a 28 byte identifier that uniquely identifies a namespace. T ## Reserved Namespaces -| name | type | value | description | -|-------------------------------------|-------------|----------------------------------------------------------------|------------------------------------------------------------------------------------------------------| -| `TRANSACTION_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000001` | Transactions: requests that modify the state. | -| `INTERMEDIATE_STATE_ROOT_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000002` | Intermediate state roots, committed after every transaction. | -| `PAY_FOR_BLOB_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000004` | Namespace reserved for transactions that contain a PayForBlob. | -| `RESERVED_PADDING_NAMESPACE` | `Namespace` | `0x00000000000000000000000000000000000000000000000000000000FF` | Padding after all reserved namespaces but before blobs. | -| `MAX_RESERVED_NAMESPACE` | `Namespace` | `0x00000000000000000000000000000000000000000000000000000000FF` | Max reserved namespace is lexicographically the largest namespace that is reserved for protocol use. | -| `TAIL_PADDING_NAMESPACE` | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Tail padding for blobs: padding after all blobs to fill up the original data square. | -| `PARITY_SHARE_NAMESPACE` | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Parity shares: extended shares in the available data matrix. | +Celestia reserves certain namespaces with specific meanings. +Reserved namespaces are used to properly organize and order transactions and blobs inside the [data square](./data_square_layout.md). +Applications MUST NOT use these reserved namespaces for their blob data. + +Reserved namespaces fall into two categories, _Primary Reserved Namespaces_ and _Secondary Reserved Namespaces_. + +- Primary Reserved Namespaces: Namespaces with values less than or equal to `0x00000000000000000000000000000000000000000000000000000000FF` are referred to as Primary Reserved Namespaces and are exclusively reserved for use within the Celestia protocols. +- Secondary Reserved Namespaces: Currently, there are two secondary reserved namespaces, namely, `PARITY_SHARE_NAMESPACE` and `TAIL_PADDING_NAMESPACE` which fall out of the range of primary reserved namespaces. +In the table, you will notice that the `PARITY_SHARE_NAMESPACE` and `TAIL_PADDING_NAMESPACE` utilize the namespace version `255`, which differs from the supported user-specified versions. +The reason for employing version `255` for the `PARITY_SHARE_NAMESPACE` is to enable more efficient proof generation within the context of [nmt](https://github.com/celestiaorg/nmt), where it is used in conjunction with the `IgnoreMaxNamespace` feature. +Similarly, the `TAIL_PADDING_NAMESPACE` utilizes the namespace version `255` to ensure that padding shares are always properly ordered and placed at the end of the Celestia data square even if a new namespace version is introduced. + +Below is a list of currently used primary and secondary reserved namespaces, along with a brief description of each. +For additional information on the significance and application of the reserved namespaces, please refer to the [Data Square Layout](./data_square_layout.md) specifications. + +| name | type | value | description | +|-------------------------------------|-------------|----------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| +| `TRANSACTION_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000001` | Transactions: requests that modify the state. | +| `INTERMEDIATE_STATE_ROOT_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000002` | Intermediate state roots, committed after every transaction. | +| `PAY_FOR_BLOB_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000004` | Namespace reserved for transactions that contain a PayForBlob. | +| `RESERVED_PADDING_NAMESPACE` | `Namespace` | `0x00000000000000000000000000000000000000000000000000000000FF` | Padding after all reserved namespaces but before blobs. | +| `MAX_PRIMARY_RESERVED_NAMESPACE` | `Namespace` | `0x00000000000000000000000000000000000000000000000000000000FF` | Max primary reserved namespace is the largest primary reserved namespace designated for protocol use. | +| `TAIL_PADDING_NAMESPACE` | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Tail padding for blobs: padding after all blobs to fill up the original data square. | +| `PARITY_SHARE_NAMESPACE` | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Parity shares: extended shares in the available data matrix. | ## Assumptions and Considerations diff --git a/x/blob/types/payforblob.go b/x/blob/types/payforblob.go index 00bd9e8b41..3e95524d50 100644 --- a/x/blob/types/payforblob.go +++ b/x/blob/types/payforblob.go @@ -186,15 +186,7 @@ func DefaultEstimateGas(blobSizes []uint32) uint64 { // tail padding). func ValidateBlobNamespace(ns appns.Namespace) error { if ns.IsReserved() { - return ErrReservedNamespace.Wrapf("got namespace: %x, want: > %x", ns, appns.MaxReservedNamespace) - } - - if ns.IsParityShares() { - return ErrParitySharesNamespace - } - - if ns.IsTailPadding() { - return ErrTailPaddingNamespace + return ErrReservedNamespace } if !slices.Contains(appns.SupportedBlobNamespaceVersions, ns.Version) { diff --git a/x/blob/types/payforblob_test.go b/x/blob/types/payforblob_test.go index cfdaa1d844..84c79f14da 100644 --- a/x/blob/types/payforblob_test.go +++ b/x/blob/types/payforblob_test.go @@ -139,9 +139,9 @@ func TestValidateBasic(t *testing.T) { intermediateStateRootsNamespaceMsg := validMsgPayForBlobs(t) intermediateStateRootsNamespaceMsg.Namespaces[0] = appns.IntermediateStateRootsNamespace.Bytes() - // MsgPayForBlobs that uses the max reserved namespace + // MsgPayForBlobs that uses the max primary reserved namespace maxReservedNamespaceMsg := validMsgPayForBlobs(t) - maxReservedNamespaceMsg.Namespaces[0] = appns.MaxReservedNamespace.Bytes() + maxReservedNamespaceMsg.Namespaces[0] = appns.MaxPrimaryReservedNamespace.Bytes() // MsgPayForBlobs that has an empty share commitment emptyShareCommitment := validMsgPayForBlobs(t) @@ -176,12 +176,12 @@ func TestValidateBasic(t *testing.T) { { name: "parity shares namespace", msg: paritySharesMsg, - wantErr: ErrParitySharesNamespace, + wantErr: ErrReservedNamespace, }, { name: "tail padding namespace", msg: tailPaddingMsg, - wantErr: ErrTailPaddingNamespace, + wantErr: ErrReservedNamespace, }, { name: "tx namespace", From 2b9a883587cbbd8a525c7c6e92da22d86be1f04e Mon Sep 17 00:00:00 2001 From: Rootul P Date: Mon, 24 Jul 2023 09:21:36 -0400 Subject: [PATCH 5/6] refactor: use blob share version for padding shares (#2148) Closes https://github.com/celestiaorg/celestia-app/issues/2142 --- pkg/shares/padding.go | 21 ++++++++------- pkg/shares/padding_test.go | 4 +-- pkg/shares/parse_test.go | 2 +- pkg/shares/share_sequence_test.go | 2 +- pkg/shares/shares_test.go | 2 +- pkg/shares/split_sparse_shares.go | 6 ++++- pkg/shares/split_sparse_shares_test.go | 36 ++++++++++++++++++++++++++ 7 files changed, 57 insertions(+), 16 deletions(-) diff --git a/pkg/shares/padding.go b/pkg/shares/padding.go index d2ee22bb73..aa5c2e425c 100644 --- a/pkg/shares/padding.go +++ b/pkg/shares/padding.go @@ -10,10 +10,11 @@ import ( // NamespacePaddingShare returns a share that acts as padding. Namespace padding // shares follow a blob so that the next blob may start at an index that -// conforms to blob share commitment rules. The ns parameter provided should -// be the namespace of the blob that precedes this padding in the data square. -func NamespacePaddingShare(ns appns.Namespace) (Share, error) { - b, err := NewBuilder(ns, appconsts.ShareVersionZero, true).Init() +// conforms to blob share commitment rules. The ns and shareVersion parameters +// provided should be the namespace and shareVersion of the blob that precedes +// this padding in the data square. +func NamespacePaddingShare(ns appns.Namespace, shareVersion uint8) (Share, error) { + b, err := NewBuilder(ns, shareVersion, true).Init() if err != nil { return Share{}, err } @@ -32,14 +33,14 @@ func NamespacePaddingShare(ns appns.Namespace) (Share, error) { } // NamespacePaddingShares returns n namespace padding shares. -func NamespacePaddingShares(ns appns.Namespace, n int) ([]Share, error) { +func NamespacePaddingShares(ns appns.Namespace, shareVersion uint8, n int) ([]Share, error) { var err error if n < 0 { return nil, errors.New("n must be positive") } shares := make([]Share, n) for i := 0; i < n; i++ { - shares[i], err = NamespacePaddingShare(ns) + shares[i], err = NamespacePaddingShare(ns, shareVersion) if err != nil { return shares, err } @@ -52,7 +53,7 @@ func NamespacePaddingShares(ns appns.Namespace, n int) ([]Share, error) { // first blob can start at an index that conforms to non-interactive default // rules. func ReservedPaddingShare() Share { - share, err := NamespacePaddingShare(appns.ReservedPaddingNamespace) + share, err := NamespacePaddingShare(appns.ReservedPaddingNamespace, appconsts.ShareVersionZero) if err != nil { panic(err) } @@ -61,7 +62,7 @@ func ReservedPaddingShare() Share { // ReservedPaddingShare returns n reserved padding shares. func ReservedPaddingShares(n int) []Share { - shares, err := NamespacePaddingShares(appns.ReservedPaddingNamespace, n) + shares, err := NamespacePaddingShares(appns.ReservedPaddingNamespace, appconsts.ShareVersionZero, n) if err != nil { panic(err) } @@ -72,7 +73,7 @@ func ReservedPaddingShares(n int) []Share { // square size. Tail padding shares follow the last blob share in the data // square. func TailPaddingShare() Share { - share, err := NamespacePaddingShare(appns.TailPaddingNamespace) + share, err := NamespacePaddingShare(appns.TailPaddingNamespace, appconsts.ShareVersionZero) if err != nil { panic(err) } @@ -81,7 +82,7 @@ func TailPaddingShare() Share { // TailPaddingShares returns n tail padding shares. func TailPaddingShares(n int) []Share { - shares, err := NamespacePaddingShares(appns.TailPaddingNamespace, n) + shares, err := NamespacePaddingShares(appns.TailPaddingNamespace, appconsts.ShareVersionZero, n) if err != nil { panic(err) } diff --git a/pkg/shares/padding_test.go b/pkg/shares/padding_test.go index abf04e8515..0d0317953e 100644 --- a/pkg/shares/padding_test.go +++ b/pkg/shares/padding_test.go @@ -40,13 +40,13 @@ var tailPadding, _ = zeroPadIfNecessary( ), appconsts.ShareSize) func TestNamespacePaddingShare(t *testing.T) { - got, err := NamespacePaddingShare(ns1) + got, err := NamespacePaddingShare(ns1, appconsts.ShareVersionZero) assert.NoError(t, err) assert.Equal(t, nsOnePadding, got.ToBytes()) } func TestNamespacePaddingShares(t *testing.T) { - shares, err := NamespacePaddingShares(ns1, 2) + shares, err := NamespacePaddingShares(ns1, appconsts.ShareVersionZero, 2) assert.NoError(t, err) for _, share := range shares { assert.Equal(t, nsOnePadding, share.ToBytes()) diff --git a/pkg/shares/parse_test.go b/pkg/shares/parse_test.go index 402b8714b7..152d262658 100644 --- a/pkg/shares/parse_test.go +++ b/pkg/shares/parse_test.go @@ -41,7 +41,7 @@ func TestParseShares(t *testing.T) { // because it takes more than one share to store a sequence of 1000 bytes tooLargeSequenceLen := generateRawShare(t, ns1, true, uint32(1000)) - ns1Padding, err := NamespacePaddingShare(ns1) + ns1Padding, err := NamespacePaddingShare(ns1, appconsts.ShareVersionZero) require.NoError(t, err) type testCase struct { diff --git a/pkg/shares/share_sequence_test.go b/pkg/shares/share_sequence_test.go index 31d790cdb4..3e98304217 100644 --- a/pkg/shares/share_sequence_test.go +++ b/pkg/shares/share_sequence_test.go @@ -151,7 +151,7 @@ func Test_validSequenceLen(t *testing.T) { } ns1 := appns.MustNewV0(bytes.Repeat([]byte{0x1}, appns.NamespaceVersionZeroIDSize)) - share, err := NamespacePaddingShare(ns1) + share, err := NamespacePaddingShare(ns1, appconsts.ShareVersionZero) require.NoError(t, err) namespacePadding := ShareSequence{ Namespace: ns1, diff --git a/pkg/shares/shares_test.go b/pkg/shares/shares_test.go index 10a277e7d4..cc7ee1bb19 100644 --- a/pkg/shares/shares_test.go +++ b/pkg/shares/shares_test.go @@ -241,7 +241,7 @@ func TestIsPadding(t *testing.T) { ), appconsts.ShareSize) - nsPadding, err := NamespacePaddingShare(ns1) + nsPadding, err := NamespacePaddingShare(ns1, appconsts.ShareVersionZero) require.NoError(t, err) testCases := []testCase{ diff --git a/pkg/shares/split_sparse_shares.go b/pkg/shares/split_sparse_shares.go index 0a54ffd7f1..eb3d8a1d57 100644 --- a/pkg/shares/split_sparse_shares.go +++ b/pkg/shares/split_sparse_shares.go @@ -86,7 +86,11 @@ func (sss *SparseShareSplitter) WriteNamespacePaddingShares(count int) error { if err != nil { return err } - nsPaddingShares, err := NamespacePaddingShares(lastBlobNs, count) + lastBlobInfo, err := lastBlob.InfoByte() + if err != nil { + return err + } + nsPaddingShares, err := NamespacePaddingShares(lastBlobNs, lastBlobInfo.Version(), count) if err != nil { return err } diff --git a/pkg/shares/split_sparse_shares_test.go b/pkg/shares/split_sparse_shares_test.go index db4ff394c8..ca3c55853c 100644 --- a/pkg/shares/split_sparse_shares_test.go +++ b/pkg/shares/split_sparse_shares_test.go @@ -4,6 +4,7 @@ import ( "bytes" "testing" + "github.com/celestiaorg/celestia-app/pkg/appconsts" appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/stretchr/testify/assert" coretypes "github.com/tendermint/tendermint/types" @@ -38,3 +39,38 @@ func TestSparseShareSplitter(t *testing.T) { got := sss.Export() assert.Len(t, got, 2) } + +func TestWriteNamespacePaddingShares(t *testing.T) { + ns1 := appns.MustNewV0(bytes.Repeat([]byte{1}, appns.NamespaceVersionZeroIDSize)) + blob1 := newBlob(ns1, appconsts.ShareVersionZero) + + sss := NewSparseShareSplitter() + + err := sss.Write(blob1) + assert.NoError(t, err) + err = sss.WriteNamespacePaddingShares(1) + assert.NoError(t, err) + + // got is expected to be [blob1, padding] + got := sss.Export() + assert.Len(t, got, 2) + + // verify that the second share is padding + isPadding, err := got[1].IsPadding() + assert.NoError(t, err) + assert.True(t, isPadding) + + // verify that the padding share has the same share version as blob1 + info, err := got[1].InfoByte() + assert.NoError(t, err) + assert.Equal(t, info.Version(), appconsts.ShareVersionZero) +} + +func newBlob(ns appns.Namespace, shareVersion uint8) coretypes.Blob { + return coretypes.Blob{ + NamespaceVersion: ns.Version, + NamespaceID: ns.ID, + ShareVersion: shareVersion, + Data: []byte("data"), + } +} From 8bd17d3042b492539ef19ce12d3846edaa1b0dfb Mon Sep 17 00:00:00 2001 From: Rootul P Date: Mon, 14 Aug 2023 10:29:36 -0400 Subject: [PATCH 6/6] feat!: reserve the last 256 namespaces for protocol use (#2257) Closes https://github.com/celestiaorg/celestia-app/issues/2253 --- pkg/namespace/consts.go | 51 +++++++++++------- pkg/namespace/namespace.go | 17 +++--- pkg/namespace/namespace_test.go | 89 +++++++++++++++++++++++++++++++ pkg/shares/padding.go | 4 +- pkg/shares/padding_test.go | 2 +- pkg/shares/parse_test.go | 4 +- pkg/shares/share_sequence_test.go | 2 +- pkg/shares/shares.go | 8 +-- specs/src/specs/namespace.md | 45 ++++++++-------- specs/src/specs/shares.md | 10 ++-- 10 files changed, 169 insertions(+), 63 deletions(-) diff --git a/pkg/namespace/consts.go b/pkg/namespace/consts.go index df30ba4e21..9c930cc572 100644 --- a/pkg/namespace/consts.go +++ b/pkg/namespace/consts.go @@ -36,39 +36,50 @@ var ( // NamespaceVersionZeroPrefix is the prefix of a namespace ID for version 0. NamespaceVersionZeroPrefix = bytes.Repeat([]byte{0}, NamespaceVersionZeroPrefixSize) - // TxNamespace is the namespace reserved for transaction data. - TxNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 1}) + // TxNamespace is the namespace reserved for ordinary Cosmos SDK transactions. + TxNamespace = primaryReservedNamespace(0x01) // IntermediateStateRootsNamespace is the namespace reserved for // intermediate state root data. - IntermediateStateRootsNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 2}) + IntermediateStateRootsNamespace = primaryReservedNamespace(0x02) // PayForBlobNamespace is the namespace reserved for PayForBlobs transactions. - PayForBlobNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 4}) + PayForBlobNamespace = primaryReservedNamespace(0x04) - // ReservedPaddingNamespace is the namespace used for padding after all - // reserved namespaces. In practice this padding is after transactions - // (ordinary and PFBs) but before blobs. - ReservedPaddingNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) + // PrimaryReservedPaddingNamespace is the namespace used for padding after all + // primary reserved namespaces. + PrimaryReservedPaddingNamespace = primaryReservedNamespace(0xFF) - // MaxPrimaryReservedNamespace represents the largest primary reserved - // namespace reserved for protocol use. Note that there may be other - // non-primary reserved namespaces beyond this upper limit. - MaxPrimaryReservedNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) + // MaxPrimaryReservedNamespace is the highest primary reserved namespace. + // Namespaces lower than this are reserved for protocol use. + MaxPrimaryReservedNamespace = primaryReservedNamespace(0xFF) + + // MinSecondaryReservedNamespace is the lowest secondary reserved namespace + // reserved for protocol use. Namespaces higher than this are reserved for + // protocol use. + MinSecondaryReservedNamespace = secondaryReservedNamespace(0x00) // TailPaddingNamespace is the namespace reserved for tail padding. All data // with this namespace will be ignored. - TailPaddingNamespace = Namespace{ - Version: math.MaxUint8, - ID: append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), 0xFE), - } + TailPaddingNamespace = secondaryReservedNamespace(0xFE) // ParitySharesNamespace is the namespace reserved for erasure coded data. - ParitySharesNamespace = Namespace{ - Version: math.MaxUint8, - ID: bytes.Repeat([]byte{0xFF}, NamespaceIDSize), - } + ParitySharesNamespace = secondaryReservedNamespace(0xFF) // SupportedBlobNamespaceVersions is a list of namespace versions that can be specified by a user for blobs. SupportedBlobNamespaceVersions = []uint8{NamespaceVersionZero} ) + +func primaryReservedNamespace(lastByte byte) Namespace { + return Namespace{ + Version: NamespaceVersionZero, + ID: append(bytes.Repeat([]byte{0x00}, NamespaceIDSize-1), lastByte), + } +} + +func secondaryReservedNamespace(lastByte byte) Namespace { + return Namespace{ + Version: NamespaceVersionMax, + ID: append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), lastByte), + } +} diff --git a/pkg/namespace/namespace.go b/pkg/namespace/namespace.go index d3ea522480..126772774b 100644 --- a/pkg/namespace/namespace.go +++ b/pkg/namespace/namespace.go @@ -106,10 +106,15 @@ func validateID(version uint8, id []byte) error { // IsReserved returns true if the namespace is reserved for protocol-use. func (n Namespace) IsReserved() bool { - isLessThanOrEqualToMaxPrimaryReservedNamespace := bytes.Compare(n.Bytes(), MaxPrimaryReservedNamespace.Bytes()) < 1 - isParityNamespace := n.IsParityShares() - isTailPadding := n.IsTailPadding() - return isLessThanOrEqualToMaxPrimaryReservedNamespace || isParityNamespace || isTailPadding + return n.IsPrimaryReserved() || n.IsSecondaryReserved() +} + +func (n Namespace) IsPrimaryReserved() bool { + return n.IsLessOrEqualThan(MaxPrimaryReservedNamespace) +} + +func (n Namespace) IsSecondaryReserved() bool { + return n.IsGreaterOrEqualThan(MinSecondaryReservedNamespace) } func (n Namespace) IsParityShares() bool { @@ -120,8 +125,8 @@ func (n Namespace) IsTailPadding() bool { return bytes.Equal(n.Bytes(), TailPaddingNamespace.Bytes()) } -func (n Namespace) IsReservedPadding() bool { - return bytes.Equal(n.Bytes(), ReservedPaddingNamespace.Bytes()) +func (n Namespace) IsPrimaryReservedPadding() bool { + return bytes.Equal(n.Bytes(), PrimaryReservedPaddingNamespace.Bytes()) } func (n Namespace) IsTx() bool { diff --git a/pkg/namespace/namespace_test.go b/pkg/namespace/namespace_test.go index f7b97487dc..8dd0794a23 100644 --- a/pkg/namespace/namespace_test.go +++ b/pkg/namespace/namespace_test.go @@ -2,6 +2,8 @@ package namespace import ( "bytes" + "math" + "reflect" "testing" "github.com/stretchr/testify/assert" @@ -195,3 +197,90 @@ func TestBytes(t *testing.T) { assert.Equal(t, want, got) } + +func TestLeftPad(t *testing.T) { + tests := []struct { + input []byte + size int + expected []byte + }{ + // input smaller than pad size + {[]byte{1, 2, 3}, 10, []byte{0, 0, 0, 0, 0, 0, 0, 1, 2, 3}}, + {[]byte{1}, 5, []byte{0, 0, 0, 0, 1}}, + {[]byte{1, 2}, 4, []byte{0, 0, 1, 2}}, + + // input equal to pad size + {[]byte{1, 2, 3}, 3, []byte{1, 2, 3}}, + {[]byte{1, 2, 3, 4}, 4, []byte{1, 2, 3, 4}}, + + // input larger than pad size + {[]byte{1, 2, 3, 4, 5}, 4, []byte{1, 2, 3, 4, 5}}, + {[]byte{1, 2, 3, 4, 5, 6, 7}, 3, []byte{1, 2, 3, 4, 5, 6, 7}}, + + // input size 0 + {[]byte{}, 8, []byte{0, 0, 0, 0, 0, 0, 0, 0}}, + {[]byte{}, 0, []byte{}}, + } + + for _, test := range tests { + result := leftPad(test.input, test.size) + assert.True(t, reflect.DeepEqual(result, test.expected)) + } +} + +func TestIsReserved(t *testing.T) { + type testCase struct { + ns Namespace + want bool + } + testCases := []testCase{ + { + ns: MustNewV0(bytes.Repeat([]byte{1}, NamespaceVersionZeroIDSize)), + want: false, + }, + { + ns: TxNamespace, + want: true, + }, + { + ns: IntermediateStateRootsNamespace, + want: true, + }, + { + ns: PayForBlobNamespace, + want: true, + }, + { + ns: PrimaryReservedPaddingNamespace, + want: true, + }, + { + ns: MaxPrimaryReservedNamespace, + want: true, + }, + { + ns: MinSecondaryReservedNamespace, + want: true, + }, + { + ns: TailPaddingNamespace, + want: true, + }, + { + ns: ParitySharesNamespace, + want: true, + }, + { + ns: Namespace{ + Version: math.MaxUint8, + ID: append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), 1), + }, + want: true, + }, + } + + for _, tc := range testCases { + got := tc.ns.IsReserved() + assert.Equal(t, tc.want, got) + } +} diff --git a/pkg/shares/padding.go b/pkg/shares/padding.go index aa5c2e425c..7283b17c5a 100644 --- a/pkg/shares/padding.go +++ b/pkg/shares/padding.go @@ -53,7 +53,7 @@ func NamespacePaddingShares(ns appns.Namespace, shareVersion uint8, n int) ([]Sh // first blob can start at an index that conforms to non-interactive default // rules. func ReservedPaddingShare() Share { - share, err := NamespacePaddingShare(appns.ReservedPaddingNamespace, appconsts.ShareVersionZero) + share, err := NamespacePaddingShare(appns.PrimaryReservedPaddingNamespace, appconsts.ShareVersionZero) if err != nil { panic(err) } @@ -62,7 +62,7 @@ func ReservedPaddingShare() Share { // ReservedPaddingShare returns n reserved padding shares. func ReservedPaddingShares(n int) []Share { - shares, err := NamespacePaddingShares(appns.ReservedPaddingNamespace, appconsts.ShareVersionZero, n) + shares, err := NamespacePaddingShares(appns.PrimaryReservedPaddingNamespace, appconsts.ShareVersionZero, n) if err != nil { panic(err) } diff --git a/pkg/shares/padding_test.go b/pkg/shares/padding_test.go index 0d0317953e..e660288aae 100644 --- a/pkg/shares/padding_test.go +++ b/pkg/shares/padding_test.go @@ -23,7 +23,7 @@ var nsOnePadding, _ = zeroPadIfNecessary( var reservedPadding, _ = zeroPadIfNecessary( append( - appns.ReservedPaddingNamespace.Bytes(), + appns.PrimaryReservedPaddingNamespace.Bytes(), []byte{ 1, // info byte 0, 0, 0, 0, // sequence len diff --git a/pkg/shares/parse_test.go b/pkg/shares/parse_test.go index 152d262658..9e26f2fc75 100644 --- a/pkg/shares/parse_test.go +++ b/pkg/shares/parse_test.go @@ -162,11 +162,11 @@ func TestParseShares(t *testing.T) { ignorePadding: false, want: []ShareSequence{ { - Namespace: appns.ReservedPaddingNamespace, + Namespace: appns.PrimaryReservedPaddingNamespace, Shares: []Share{ReservedPaddingShare()}, }, { - Namespace: appns.ReservedPaddingNamespace, + Namespace: appns.PrimaryReservedPaddingNamespace, Shares: []Share{ReservedPaddingShare()}, }, }, diff --git a/pkg/shares/share_sequence_test.go b/pkg/shares/share_sequence_test.go index 3e98304217..82a781e765 100644 --- a/pkg/shares/share_sequence_test.go +++ b/pkg/shares/share_sequence_test.go @@ -159,7 +159,7 @@ func Test_validSequenceLen(t *testing.T) { } reservedPadding := ShareSequence{ - Namespace: appns.ReservedPaddingNamespace, + Namespace: appns.PrimaryReservedPaddingNamespace, Shares: []Share{ReservedPaddingShare()}, } diff --git a/pkg/shares/shares.go b/pkg/shares/shares.go index 1f805d4e17..93711cc639 100644 --- a/pkg/shares/shares.go +++ b/pkg/shares/shares.go @@ -121,11 +121,11 @@ func (s *Share) IsPadding() (bool, error) { if err != nil { return false, err } - isReservedPadding, err := s.isReservedPadding() + isPrimaryReservedPadding, err := s.isPrimaryReservedPadding() if err != nil { return false, err } - return isNamespacePadding || isTailPadding || isReservedPadding, nil + return isNamespacePadding || isTailPadding || isPrimaryReservedPadding, nil } func (s *Share) isNamespacePadding() (bool, error) { @@ -149,12 +149,12 @@ func (s *Share) isTailPadding() (bool, error) { return ns.IsTailPadding(), nil } -func (s *Share) isReservedPadding() (bool, error) { +func (s *Share) isPrimaryReservedPadding() (bool, error) { ns, err := s.Namespace() if err != nil { return false, err } - return ns.IsReservedPadding(), nil + return ns.IsPrimaryReservedPadding(), nil } func (s *Share) ToBytes() []byte { diff --git a/specs/src/specs/namespace.md b/specs/src/specs/namespace.md index a56ac5a582..c56bc35194 100644 --- a/specs/src/specs/namespace.md +++ b/specs/src/specs/namespace.md @@ -42,30 +42,31 @@ The namespace ID is a 28 byte identifier that uniquely identifies a namespace. T ## Reserved Namespaces -Celestia reserves certain namespaces with specific meanings. -Reserved namespaces are used to properly organize and order transactions and blobs inside the [data square](./data_square_layout.md). -Applications MUST NOT use these reserved namespaces for their blob data. - -Reserved namespaces fall into two categories, _Primary Reserved Namespaces_ and _Secondary Reserved Namespaces_. - -- Primary Reserved Namespaces: Namespaces with values less than or equal to `0x00000000000000000000000000000000000000000000000000000000FF` are referred to as Primary Reserved Namespaces and are exclusively reserved for use within the Celestia protocols. -- Secondary Reserved Namespaces: Currently, there are two secondary reserved namespaces, namely, `PARITY_SHARE_NAMESPACE` and `TAIL_PADDING_NAMESPACE` which fall out of the range of primary reserved namespaces. -In the table, you will notice that the `PARITY_SHARE_NAMESPACE` and `TAIL_PADDING_NAMESPACE` utilize the namespace version `255`, which differs from the supported user-specified versions. -The reason for employing version `255` for the `PARITY_SHARE_NAMESPACE` is to enable more efficient proof generation within the context of [nmt](https://github.com/celestiaorg/nmt), where it is used in conjunction with the `IgnoreMaxNamespace` feature. -Similarly, the `TAIL_PADDING_NAMESPACE` utilizes the namespace version `255` to ensure that padding shares are always properly ordered and placed at the end of the Celestia data square even if a new namespace version is introduced. - -Below is a list of currently used primary and secondary reserved namespaces, along with a brief description of each. +Celestia reserves some namespaces for protocol use. +These namespaces are called "reserved namespaces". +Reserved namespaces are used to arrange the contents of the [data square](./data_square_layout.md). +Applications MUST NOT use reserved namespaces for their blob data. +Reserved namespaces fall into two categories: _Primary_ and _Secondary_. + +- Primary: Namespaces with values less than or equal to `0x00000000000000000000000000000000000000000000000000000000FF`. Primary namespaces always have a version of `0`. +- Secondary: Namespaces with values greater than or equal to `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00`. +Secondary namespaces always have a version of `255` (`0xFF`) so that they are placed after all user specifiable namespaces in a sorted data square. +The `PARITY_SHARE_NAMESPACE` uses version `255` (`0xFF`) to enable more efficient proof generation within the context of [nmt](https://github.com/celestiaorg/nmt), where it is used in conjunction with the `IgnoreMaxNamespace` feature. +The `TAIL_PADDING_NAMESPACE` uses the version `255` to ensure that padding shares are always placed at the end of the Celestia data square even if a new user-specifiable version is introduced. + +Below is a list of the current reserved namespaces. For additional information on the significance and application of the reserved namespaces, please refer to the [Data Square Layout](./data_square_layout.md) specifications. -| name | type | value | description | -|-------------------------------------|-------------|----------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| -| `TRANSACTION_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000001` | Transactions: requests that modify the state. | -| `INTERMEDIATE_STATE_ROOT_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000002` | Intermediate state roots, committed after every transaction. | -| `PAY_FOR_BLOB_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000004` | Namespace reserved for transactions that contain a PayForBlob. | -| `RESERVED_PADDING_NAMESPACE` | `Namespace` | `0x00000000000000000000000000000000000000000000000000000000FF` | Padding after all reserved namespaces but before blobs. | -| `MAX_PRIMARY_RESERVED_NAMESPACE` | `Namespace` | `0x00000000000000000000000000000000000000000000000000000000FF` | Max primary reserved namespace is the largest primary reserved namespace designated for protocol use. | -| `TAIL_PADDING_NAMESPACE` | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Tail padding for blobs: padding after all blobs to fill up the original data square. | -| `PARITY_SHARE_NAMESPACE` | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Parity shares: extended shares in the available data matrix. | +| name | type | category | value | description | +|--------------------------------------|-------------|-----------|----------------------------------------------------------------|----------------------------------------------------------------------------| +| `TRANSACTION_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000001` | Namespace for ordinary Cosmos SDK transactions. | +| `INTERMEDIATE_STATE_ROOT_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000002` | Namespace for intermediate state roots (not currently utilized). | +| `PAY_FOR_BLOB_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000004` | Namespace for transactions that contain a PayForBlob. | +| `PRIMARY_RESERVED_PADDING_NAMESPACE` | `Namespace` | Primary | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for padding after all primary reserved namespaces. | +| `MAX_PRIMARY_RESERVED_NAMESPACE` | `Namespace` | Primary | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for the highest primary reserved namespace. | +| `MIN_SECONDARY_RESERVED_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00` | Namespace for the lowest secondary reserved namespace. | +| `TAIL_PADDING_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Namespace for padding after all blobs to fill up the original data square. | +| `PARITY_SHARE_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Namespace for parity shares. | ## Assumptions and Considerations diff --git a/specs/src/specs/shares.md b/specs/src/specs/shares.md index 1c9fcfb2e4..cfa57e2f25 100644 --- a/specs/src/specs/shares.md +++ b/specs/src/specs/shares.md @@ -18,7 +18,7 @@ User submitted transactions are split into shares (see [share splitting](#share- [Padding](#padding) shares are added to the `k * k` matrix to ensure: -1. Blob sequences start on an index that conforms to [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) (see [namespace padding share](#namespace-padding-share) and [reserved padding share](#reserved-padding-share)) +1. Blob sequences start on an index that conforms to [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) (see [namespace padding share](#namespace-padding-share) and [reserved padding share](#primary-reserved-padding-share)) 1. The number of shares in the matrix is a perfect square (see [tail padding share](#tail-padding-share)) ## Share Format @@ -85,17 +85,17 @@ Padding shares vary based on namespace but they conform to the [share format](#s A namespace padding share uses the namespace of the blob that precedes it in the data square so that the data square can retain the property that all shares are ordered by namespace. A namespace padding share acts as padding between blobs so that the subsequent blob begins at an index that conforms to the [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules). Clients MAY ignore the contents of these shares because they don't contain any significant data. -### Reserved Padding Share +### Primary Reserved Padding Share -Reserved padding shares use the [`RESERVED_PADDING_NAMESPACE`](./consensus.md#constants). Reserved padding shares are placed after the last reserved namespace share in the data square so that the first blob can start at an index that conforms to blob share commitment rules. Clients MAY ignore the contents of these shares because they don't contain any significant data. +Primary reserved padding shares use the [`PRIMARY_RESERVED_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Primary reserved padding shares are placed after shares in the primary reserved namespace range so that the first blob can start at an index that conforms to blob share commitment rules. Clients MAY ignore the contents of these shares because they don't contain any significant data. ### Tail Padding Share -Tail padding shares use the [`TAIL_PADDING_NAMESPACE`](./consensus.md#constants). Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients MAY ignore the contents of these shares because they don't contain any significant data. +Tail padding shares use the [`TAIL_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients MAY ignore the contents of these shares because they don't contain any significant data. ## Parity Share -Parity shares use the [`PARITY_SHARE_NAMESPACE`](./consensus.md#constants). Parity shares are the output of the erasure coding step of the data square construction process. They occupy quadrants Q1, Q2, and Q3 of the extended data square and are used to reconstruct the original data square (Q0). Bytes carry no special meaning. +Parity shares use the [`PARITY_SHARE_NAMESPACE`](./namespace.md#reserved-namespaces). Parity shares are the output of the erasure coding step of the data square construction process. They occupy quadrants Q1, Q2, and Q3 of the extended data square and are used to reconstruct the original data square (Q0). Bytes carry no special meaning. ## Share Splitting