From cb8247ead807a52bb8915d450982544755efc1e6 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Mon, 14 Aug 2023 10:29:36 -0400 Subject: [PATCH] feat!: reserve the last 256 namespaces for protocol use (#2257) Closes https://github.com/celestiaorg/celestia-app/issues/2253 --- pkg/namespace/consts.go | 51 ++++++++++++++++----------- pkg/namespace/namespace.go | 17 +++++---- pkg/namespace/namespace_test.go | 58 +++++++++++++++++++++++++++++++ pkg/shares/padding.go | 4 +-- pkg/shares/padding_test.go | 2 +- pkg/shares/parse_test.go | 4 +-- pkg/shares/share_sequence_test.go | 2 +- pkg/shares/shares.go | 8 ++--- specs/src/specs/namespace.md | 45 ++++++++++++------------ specs/src/specs/shares.md | 10 +++--- 10 files changed, 138 insertions(+), 63 deletions(-) diff --git a/pkg/namespace/consts.go b/pkg/namespace/consts.go index df30ba4e21..9c930cc572 100644 --- a/pkg/namespace/consts.go +++ b/pkg/namespace/consts.go @@ -36,39 +36,50 @@ var ( // NamespaceVersionZeroPrefix is the prefix of a namespace ID for version 0. NamespaceVersionZeroPrefix = bytes.Repeat([]byte{0}, NamespaceVersionZeroPrefixSize) - // TxNamespace is the namespace reserved for transaction data. - TxNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 1}) + // TxNamespace is the namespace reserved for ordinary Cosmos SDK transactions. + TxNamespace = primaryReservedNamespace(0x01) // IntermediateStateRootsNamespace is the namespace reserved for // intermediate state root data. - IntermediateStateRootsNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 2}) + IntermediateStateRootsNamespace = primaryReservedNamespace(0x02) // PayForBlobNamespace is the namespace reserved for PayForBlobs transactions. - PayForBlobNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 4}) + PayForBlobNamespace = primaryReservedNamespace(0x04) - // ReservedPaddingNamespace is the namespace used for padding after all - // reserved namespaces. In practice this padding is after transactions - // (ordinary and PFBs) but before blobs. - ReservedPaddingNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) + // PrimaryReservedPaddingNamespace is the namespace used for padding after all + // primary reserved namespaces. + PrimaryReservedPaddingNamespace = primaryReservedNamespace(0xFF) - // MaxPrimaryReservedNamespace represents the largest primary reserved - // namespace reserved for protocol use. Note that there may be other - // non-primary reserved namespaces beyond this upper limit. - MaxPrimaryReservedNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) + // MaxPrimaryReservedNamespace is the highest primary reserved namespace. + // Namespaces lower than this are reserved for protocol use. + MaxPrimaryReservedNamespace = primaryReservedNamespace(0xFF) + + // MinSecondaryReservedNamespace is the lowest secondary reserved namespace + // reserved for protocol use. Namespaces higher than this are reserved for + // protocol use. + MinSecondaryReservedNamespace = secondaryReservedNamespace(0x00) // TailPaddingNamespace is the namespace reserved for tail padding. All data // with this namespace will be ignored. - TailPaddingNamespace = Namespace{ - Version: math.MaxUint8, - ID: append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), 0xFE), - } + TailPaddingNamespace = secondaryReservedNamespace(0xFE) // ParitySharesNamespace is the namespace reserved for erasure coded data. - ParitySharesNamespace = Namespace{ - Version: math.MaxUint8, - ID: bytes.Repeat([]byte{0xFF}, NamespaceIDSize), - } + ParitySharesNamespace = secondaryReservedNamespace(0xFF) // SupportedBlobNamespaceVersions is a list of namespace versions that can be specified by a user for blobs. SupportedBlobNamespaceVersions = []uint8{NamespaceVersionZero} ) + +func primaryReservedNamespace(lastByte byte) Namespace { + return Namespace{ + Version: NamespaceVersionZero, + ID: append(bytes.Repeat([]byte{0x00}, NamespaceIDSize-1), lastByte), + } +} + +func secondaryReservedNamespace(lastByte byte) Namespace { + return Namespace{ + Version: NamespaceVersionMax, + ID: append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), lastByte), + } +} diff --git a/pkg/namespace/namespace.go b/pkg/namespace/namespace.go index 682385721f..3186d78509 100644 --- a/pkg/namespace/namespace.go +++ b/pkg/namespace/namespace.go @@ -106,10 +106,15 @@ func validateID(version uint8, id []byte) error { // IsReserved returns true if the namespace is reserved for protocol-use. func (n Namespace) IsReserved() bool { - isLessThanOrEqualToMaxPrimaryReservedNamespace := bytes.Compare(n.Bytes(), MaxPrimaryReservedNamespace.Bytes()) < 1 - isParityNamespace := n.IsParityShares() - isTailPadding := n.IsTailPadding() - return isLessThanOrEqualToMaxPrimaryReservedNamespace || isParityNamespace || isTailPadding + return n.IsPrimaryReserved() || n.IsSecondaryReserved() +} + +func (n Namespace) IsPrimaryReserved() bool { + return n.IsLessOrEqualThan(MaxPrimaryReservedNamespace) +} + +func (n Namespace) IsSecondaryReserved() bool { + return n.IsGreaterOrEqualThan(MinSecondaryReservedNamespace) } func (n Namespace) IsParityShares() bool { @@ -120,8 +125,8 @@ func (n Namespace) IsTailPadding() bool { return bytes.Equal(n.Bytes(), TailPaddingNamespace.Bytes()) } -func (n Namespace) IsReservedPadding() bool { - return bytes.Equal(n.Bytes(), ReservedPaddingNamespace.Bytes()) +func (n Namespace) IsPrimaryReservedPadding() bool { + return bytes.Equal(n.Bytes(), PrimaryReservedPaddingNamespace.Bytes()) } func (n Namespace) IsTx() bool { diff --git a/pkg/namespace/namespace_test.go b/pkg/namespace/namespace_test.go index fcda8472d5..57c125cc6c 100644 --- a/pkg/namespace/namespace_test.go +++ b/pkg/namespace/namespace_test.go @@ -2,6 +2,7 @@ package namespace import ( "bytes" + "math" "reflect" "testing" @@ -239,3 +240,60 @@ func TestLeftPad(t *testing.T) { assert.True(t, reflect.DeepEqual(result, test.expected)) } } + +func TestIsReserved(t *testing.T) { + type testCase struct { + ns Namespace + want bool + } + testCases := []testCase{ + { + ns: MustNewV0(bytes.Repeat([]byte{1}, NamespaceVersionZeroIDSize)), + want: false, + }, + { + ns: TxNamespace, + want: true, + }, + { + ns: IntermediateStateRootsNamespace, + want: true, + }, + { + ns: PayForBlobNamespace, + want: true, + }, + { + ns: PrimaryReservedPaddingNamespace, + want: true, + }, + { + ns: MaxPrimaryReservedNamespace, + want: true, + }, + { + ns: MinSecondaryReservedNamespace, + want: true, + }, + { + ns: TailPaddingNamespace, + want: true, + }, + { + ns: ParitySharesNamespace, + want: true, + }, + { + ns: Namespace{ + Version: math.MaxUint8, + ID: append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), 1), + }, + want: true, + }, + } + + for _, tc := range testCases { + got := tc.ns.IsReserved() + assert.Equal(t, tc.want, got) + } +} diff --git a/pkg/shares/padding.go b/pkg/shares/padding.go index 6f68378ebc..b23d1dabc4 100644 --- a/pkg/shares/padding.go +++ b/pkg/shares/padding.go @@ -53,7 +53,7 @@ func NamespacePaddingShares(ns appns.Namespace, shareVersion uint8, n int) ([]Sh // first blob can start at an index that conforms to non-interactive default // rules. func ReservedPaddingShare() Share { - share, err := NamespacePaddingShare(appns.ReservedPaddingNamespace, appconsts.ShareVersionZero) + share, err := NamespacePaddingShare(appns.PrimaryReservedPaddingNamespace, appconsts.ShareVersionZero) if err != nil { panic(err) } @@ -62,7 +62,7 @@ func ReservedPaddingShare() Share { // ReservedPaddingShare returns n reserved padding shares. func ReservedPaddingShares(n int) []Share { - shares, err := NamespacePaddingShares(appns.ReservedPaddingNamespace, appconsts.ShareVersionZero, n) + shares, err := NamespacePaddingShares(appns.PrimaryReservedPaddingNamespace, appconsts.ShareVersionZero, n) if err != nil { panic(err) } diff --git a/pkg/shares/padding_test.go b/pkg/shares/padding_test.go index 0d0317953e..e660288aae 100644 --- a/pkg/shares/padding_test.go +++ b/pkg/shares/padding_test.go @@ -23,7 +23,7 @@ var nsOnePadding, _ = zeroPadIfNecessary( var reservedPadding, _ = zeroPadIfNecessary( append( - appns.ReservedPaddingNamespace.Bytes(), + appns.PrimaryReservedPaddingNamespace.Bytes(), []byte{ 1, // info byte 0, 0, 0, 0, // sequence len diff --git a/pkg/shares/parse_test.go b/pkg/shares/parse_test.go index 152d262658..9e26f2fc75 100644 --- a/pkg/shares/parse_test.go +++ b/pkg/shares/parse_test.go @@ -162,11 +162,11 @@ func TestParseShares(t *testing.T) { ignorePadding: false, want: []ShareSequence{ { - Namespace: appns.ReservedPaddingNamespace, + Namespace: appns.PrimaryReservedPaddingNamespace, Shares: []Share{ReservedPaddingShare()}, }, { - Namespace: appns.ReservedPaddingNamespace, + Namespace: appns.PrimaryReservedPaddingNamespace, Shares: []Share{ReservedPaddingShare()}, }, }, diff --git a/pkg/shares/share_sequence_test.go b/pkg/shares/share_sequence_test.go index 3e98304217..82a781e765 100644 --- a/pkg/shares/share_sequence_test.go +++ b/pkg/shares/share_sequence_test.go @@ -159,7 +159,7 @@ func Test_validSequenceLen(t *testing.T) { } reservedPadding := ShareSequence{ - Namespace: appns.ReservedPaddingNamespace, + Namespace: appns.PrimaryReservedPaddingNamespace, Shares: []Share{ReservedPaddingShare()}, } diff --git a/pkg/shares/shares.go b/pkg/shares/shares.go index 1f805d4e17..93711cc639 100644 --- a/pkg/shares/shares.go +++ b/pkg/shares/shares.go @@ -121,11 +121,11 @@ func (s *Share) IsPadding() (bool, error) { if err != nil { return false, err } - isReservedPadding, err := s.isReservedPadding() + isPrimaryReservedPadding, err := s.isPrimaryReservedPadding() if err != nil { return false, err } - return isNamespacePadding || isTailPadding || isReservedPadding, nil + return isNamespacePadding || isTailPadding || isPrimaryReservedPadding, nil } func (s *Share) isNamespacePadding() (bool, error) { @@ -149,12 +149,12 @@ func (s *Share) isTailPadding() (bool, error) { return ns.IsTailPadding(), nil } -func (s *Share) isReservedPadding() (bool, error) { +func (s *Share) isPrimaryReservedPadding() (bool, error) { ns, err := s.Namespace() if err != nil { return false, err } - return ns.IsReservedPadding(), nil + return ns.IsPrimaryReservedPadding(), nil } func (s *Share) ToBytes() []byte { diff --git a/specs/src/specs/namespace.md b/specs/src/specs/namespace.md index 1d3ed4c538..871e204fc5 100644 --- a/specs/src/specs/namespace.md +++ b/specs/src/specs/namespace.md @@ -57,30 +57,31 @@ The ID is encoded as a byte slice of length 28. ## Reserved Namespaces -Celestia reserves certain namespaces with specific meanings. -Reserved namespaces are used to properly organize and order transactions and blobs inside the [data square](./data_square_layout.md). -Applications MUST NOT use these reserved namespaces for their blob data. - -Reserved namespaces fall into two categories, _Primary Reserved Namespaces_ and _Secondary Reserved Namespaces_. - -- Primary Reserved Namespaces: Namespaces with values less than or equal to `0x00000000000000000000000000000000000000000000000000000000FF` are referred to as Primary Reserved Namespaces and are exclusively reserved for use within the Celestia protocols. -- Secondary Reserved Namespaces: Currently, there are two secondary reserved namespaces, namely, `PARITY_SHARE_NAMESPACE` and `TAIL_PADDING_NAMESPACE` which fall out of the range of primary reserved namespaces. -In the table, you will notice that the `PARITY_SHARE_NAMESPACE` and `TAIL_PADDING_NAMESPACE` utilize the namespace version `255`, which differs from the supported user-specified versions. -The reason for employing version `255` for the `PARITY_SHARE_NAMESPACE` is to enable more efficient proof generation within the context of [nmt](https://github.com/celestiaorg/nmt), where it is used in conjunction with the `IgnoreMaxNamespace` feature. -Similarly, the `TAIL_PADDING_NAMESPACE` utilizes the namespace version `255` to ensure that padding shares are always properly ordered and placed at the end of the Celestia data square even if a new namespace version is introduced. - -Below is a list of currently used primary and secondary reserved namespaces, along with a brief description of each. +Celestia reserves some namespaces for protocol use. +These namespaces are called "reserved namespaces". +Reserved namespaces are used to arrange the contents of the [data square](./data_square_layout.md). +Applications MUST NOT use reserved namespaces for their blob data. +Reserved namespaces fall into two categories: _Primary_ and _Secondary_. + +- Primary: Namespaces with values less than or equal to `0x00000000000000000000000000000000000000000000000000000000FF`. Primary namespaces always have a version of `0`. +- Secondary: Namespaces with values greater than or equal to `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00`. +Secondary namespaces always have a version of `255` (`0xFF`) so that they are placed after all user specifiable namespaces in a sorted data square. +The `PARITY_SHARE_NAMESPACE` uses version `255` (`0xFF`) to enable more efficient proof generation within the context of [nmt](https://github.com/celestiaorg/nmt), where it is used in conjunction with the `IgnoreMaxNamespace` feature. +The `TAIL_PADDING_NAMESPACE` uses the version `255` to ensure that padding shares are always placed at the end of the Celestia data square even if a new user-specifiable version is introduced. + +Below is a list of the current reserved namespaces. For additional information on the significance and application of the reserved namespaces, please refer to the [Data Square Layout](./data_square_layout.md) specifications. -| name | type | value | description | -|-------------------------------------|-------------|----------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| -| `TRANSACTION_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000001` | Transactions: requests that modify the state. | -| `INTERMEDIATE_STATE_ROOT_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000002` | Intermediate state roots, committed after every transaction. | -| `PAY_FOR_BLOB_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000004` | Namespace reserved for transactions that contain a PayForBlob. | -| `RESERVED_PADDING_NAMESPACE` | `Namespace` | `0x00000000000000000000000000000000000000000000000000000000FF` | Padding after all reserved namespaces but before blobs. | -| `MAX_PRIMARY_RESERVED_NAMESPACE` | `Namespace` | `0x00000000000000000000000000000000000000000000000000000000FF` | Max primary reserved namespace is the largest primary reserved namespace designated for protocol use. | -| `TAIL_PADDING_NAMESPACE` | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Tail padding for blobs: padding after all blobs to fill up the original data square. | -| `PARITY_SHARE_NAMESPACE` | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Parity shares: extended shares in the available data matrix. | +| name | type | category | value | description | +|--------------------------------------|-------------|-----------|----------------------------------------------------------------|----------------------------------------------------------------------------| +| `TRANSACTION_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000001` | Namespace for ordinary Cosmos SDK transactions. | +| `INTERMEDIATE_STATE_ROOT_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000002` | Namespace for intermediate state roots (not currently utilized). | +| `PAY_FOR_BLOB_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000004` | Namespace for transactions that contain a PayForBlob. | +| `PRIMARY_RESERVED_PADDING_NAMESPACE` | `Namespace` | Primary | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for padding after all primary reserved namespaces. | +| `MAX_PRIMARY_RESERVED_NAMESPACE` | `Namespace` | Primary | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for the highest primary reserved namespace. | +| `MIN_SECONDARY_RESERVED_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00` | Namespace for the lowest secondary reserved namespace. | +| `TAIL_PADDING_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Namespace for padding after all blobs to fill up the original data square. | +| `PARITY_SHARE_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Namespace for parity shares. | ## Assumptions and Considerations diff --git a/specs/src/specs/shares.md b/specs/src/specs/shares.md index 1c9fcfb2e4..cfa57e2f25 100644 --- a/specs/src/specs/shares.md +++ b/specs/src/specs/shares.md @@ -18,7 +18,7 @@ User submitted transactions are split into shares (see [share splitting](#share- [Padding](#padding) shares are added to the `k * k` matrix to ensure: -1. Blob sequences start on an index that conforms to [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) (see [namespace padding share](#namespace-padding-share) and [reserved padding share](#reserved-padding-share)) +1. Blob sequences start on an index that conforms to [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) (see [namespace padding share](#namespace-padding-share) and [reserved padding share](#primary-reserved-padding-share)) 1. The number of shares in the matrix is a perfect square (see [tail padding share](#tail-padding-share)) ## Share Format @@ -85,17 +85,17 @@ Padding shares vary based on namespace but they conform to the [share format](#s A namespace padding share uses the namespace of the blob that precedes it in the data square so that the data square can retain the property that all shares are ordered by namespace. A namespace padding share acts as padding between blobs so that the subsequent blob begins at an index that conforms to the [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules). Clients MAY ignore the contents of these shares because they don't contain any significant data. -### Reserved Padding Share +### Primary Reserved Padding Share -Reserved padding shares use the [`RESERVED_PADDING_NAMESPACE`](./consensus.md#constants). Reserved padding shares are placed after the last reserved namespace share in the data square so that the first blob can start at an index that conforms to blob share commitment rules. Clients MAY ignore the contents of these shares because they don't contain any significant data. +Primary reserved padding shares use the [`PRIMARY_RESERVED_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Primary reserved padding shares are placed after shares in the primary reserved namespace range so that the first blob can start at an index that conforms to blob share commitment rules. Clients MAY ignore the contents of these shares because they don't contain any significant data. ### Tail Padding Share -Tail padding shares use the [`TAIL_PADDING_NAMESPACE`](./consensus.md#constants). Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients MAY ignore the contents of these shares because they don't contain any significant data. +Tail padding shares use the [`TAIL_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients MAY ignore the contents of these shares because they don't contain any significant data. ## Parity Share -Parity shares use the [`PARITY_SHARE_NAMESPACE`](./consensus.md#constants). Parity shares are the output of the erasure coding step of the data square construction process. They occupy quadrants Q1, Q2, and Q3 of the extended data square and are used to reconstruct the original data square (Q0). Bytes carry no special meaning. +Parity shares use the [`PARITY_SHARE_NAMESPACE`](./namespace.md#reserved-namespaces). Parity shares are the output of the erasure coding step of the data square construction process. They occupy quadrants Q1, Q2, and Q3 of the extended data square and are used to reconstruct the original data square (Q0). Bytes carry no special meaning. ## Share Splitting