feat: Txs To Shares Roundtrip (rollkit#889)

Adds ability to:
- Convert `TxWithISRs` to Shares
- Convert Shares to byte array (that will be posted on celestia)
- Convert above byte array back to Shares
- Convert Shares back to `TxWithISRs`
- Parse Out of Context Shares

Adds relevant roundtrip tests

Resolves rollkit#881 
Resolves rollkit#934 
Resolves rollkit#886 
Resolves rollkit#925 

Note: All shares are written and interpreted as compact shares so they
contain reserved bytes (see
https://celestiaorg.github.io/celestia-app/specs/data_structures.html#compact-share)

Related PR in `celestia-app`:
celestiaorg/celestia-app#1770

third_party/celestia-app/shares/compact_shares_test.go

@@ -6,7 +6,6 @@ import (
 	"testing"
 	"time"
 
-	coretypes "github.com/cometbft/cometbft/types"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -15,24 +14,6 @@ import (
 	"github.com/rollkit/rollkit/third_party/celestia-app/testfactory"
 )
 
-func SplitTxs(txs coretypes.Txs) (txShares []Share, err error) {
-	txWriter := NewCompactShareSplitter(appns.TxNamespace, appconsts.ShareVersionZero)
-
-	for _, tx := range txs {
-		err = txWriter.WriteTx(tx)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	txShares, _, err = txWriter.Export(0)
-	if err != nil {
-		return nil, err
-	}
-
-	return txShares, nil
-}
-
 func TestCompactShareSplitter(t *testing.T) {
 	// note that this test is mainly for debugging purposes, the main round trip
 	// tests occur in TestMerge and Test_processCompactShares
@@ -131,6 +112,29 @@ func Test_processCompactShares(t *testing.T) {
 	}
 }
 
+func TestAllSplit(t *testing.T) {
+	txs := testfactory.GenerateRandomlySizedTxs(1000, 150)
+	txShares, err := SplitTxs(txs)
+	require.NoError(t, err)
+	resTxs, err := ParseTxs(txShares)
+	require.NoError(t, err)
+	assert.Equal(t, resTxs, txs)
+}
+
+func TestParseRandomOutOfContextShares(t *testing.T) {
+	txs := testfactory.GenerateRandomlySizedTxs(1000, 150)
+	txShares, err := SplitTxs(txs)
+	require.NoError(t, err)
+
+	for i := 0; i < 1000; i++ {
+		start, length := testfactory.GetRandomSubSlice(len(txShares))
+		randomRange := NewRange(start, start+length)
+		resTxs, err := ParseTxs(txShares[randomRange.Start:randomRange.End])
+		require.NoError(t, err)
+		assert.True(t, testfactory.CheckSubArray(txs, resTxs))
+	}
+}
+
 func TestCompactShareContainsInfoByte(t *testing.T) {
 	css := NewCompactShareSplitter(appns.TxNamespace, appconsts.ShareVersionZero)
 	txs := testfactory.GenerateRandomTxs(1, appconsts.ContinuationCompactShareContentSize/4)
@@ -197,10 +201,6 @@ func Test_parseCompactSharesErrors(t *testing.T) {
 	}
 
 	testCases := []testCase{
-		{
-			"share with start indicator false",
-			txShares[1:], // set the first share to the second share which has the start indicator set to false
-		},
 		{
 			"share with unsupported share version",
 			[]Share{*shareWithUnsupportedShareVersion},

third_party/celestia-app/shares/parse_compact_shares.go

@@ -1,6 +1,12 @@
 package shares
 
-import "errors"
+import (
+	"github.com/rollkit/rollkit/third_party/celestia-app/appconsts"
+)
+
+func ParseCompactShares(shares []Share) (data [][]byte, err error) {
+	return parseCompactShares(shares, appconsts.SupportedShareVersions)
+}
 
 // parseCompactShares returns data (transactions or intermediate state roots
 // based on the contents of rawShares and supportedShareVersions. If rawShares
@@ -13,14 +19,6 @@ func parseCompactShares(shares []Share, supportedShareVersions []uint8) (data []
 		return nil, nil
 	}
 
-	seqStart, err := shares[0].IsSequenceStart()
-	if err != nil {
-		return nil, err
-	}
-	if !seqStart {
-		return nil, errors.New("first share is not the start of a sequence")
-	}
-
 	err = validateShareVersions(shares, supportedShareVersions)
 	if err != nil {
 		return nil, err
@@ -61,19 +59,32 @@ func parseRawData(rawData []byte) (units [][]byte, err error) {
 		if err != nil {
 			return nil, err
 		}
+		// the rest of raw data is padding
 		if unitLen == 0 {
 			return units, nil
 		}
+		// the rest of actual data contains only part of the next transaction so
+		// we stop parsing raw data
+		if unitLen > uint64(len(actualData)) {
+			return units, nil
+		}
 		rawData = actualData[unitLen:]
 		units = append(units, actualData[:unitLen])
 	}
 }
 
-// extractRawData returns the raw data contained in the shares. The raw data does
-// not contain the namespace ID, info byte, sequence length, or reserved bytes.
+// extractRawData returns the raw data representing complete transactions
+// contained in the shares. The raw data does not contain the namespace, info
+// byte, sequence length, or reserved bytes. Starts reading raw data based on
+// the reserved bytes in the first share.
 func extractRawData(shares []Share) (rawData []byte, err error) {
 	for i := 0; i < len(shares); i++ {
-		raw, err := shares[i].RawData()
+		var raw []byte
+		if i == 0 {
+			raw, err = shares[i].RawDataUsingReserved()
+		} else {
+			raw, err = shares[i].RawData()
+		}
 		if err != nil {
 			return nil, err
 		}

third_party/celestia-app/shares/range.go

@@ -0,0 +1,26 @@
+package shares
+
+// Range is an end exclusive set of share indexes.
+type Range struct {
+	// Start is the index of the first share occupied by this range.
+	Start int
+	// End is the next index after the last share occupied by this range.
+	End int
+}
+
+func NewRange(start, end int) Range {
+	return Range{Start: start, End: end}
+}
+
+func EmptyRange() Range {
+	return Range{Start: 0, End: 0}
+}
+
+func (r Range) IsEmpty() bool {
+	return r.Start == 0 && r.End == 0
+}
+
+func (r *Range) Add(value int) {
+	r.Start += value
+	r.End += value
+}

third_party/celestia-app/shares/share_builder.go

@@ -123,12 +123,12 @@ func (b *Builder) indexOfInfoBytes() int {
 	return appconsts.NamespaceSize
 }
 
-// MaybeWriteReservedBytes will be a no-op if the reserved bytes
+// MaybeWriteReservedBytes will be a no-op for a compact share or if the reserved bytes
 // have already been populated. If the reserved bytes are empty, it will write
 // the location of the next unit of data to the reserved bytes.
 func (b *Builder) MaybeWriteReservedBytes() error {
 	if !b.isCompactShare {
-		return errors.New("this is not a compact share")
+		return nil
 	}
 
 	empty, err := b.isEmptyReservedBytes()

third_party/celestia-app/shares/shares.go

@@ -161,6 +161,29 @@ func (s *Share) ToBytes() []byte {
 	return s.data
 }
 
+// RawDataWithReserved returns the raw share data including the reserved bytes. The raw share data does not contain the namespace ID, info byte, or sequence length.
+func (s *Share) RawDataWithReserved() (rawData []byte, err error) {
+	if len(s.data) < s.rawDataStartIndexWithReserved() {
+		return rawData, fmt.Errorf("share %s is too short to contain raw data", s)
+	}
+
+	return s.data[s.rawDataStartIndexWithReserved():], nil
+}
+
+func (s *Share) rawDataStartIndexWithReserved() int {
+	isStart, err := s.IsSequenceStart()
+	if err != nil {
+		panic(err)
+	}
+
+	index := appconsts.NamespaceSize + appconsts.ShareInfoBytes
+	if isStart {
+		index += appconsts.SequenceLenBytes
+	}
+
+	return index
+}
+
 // RawData returns the raw share data. The raw share data does not contain the
 // namespace ID, info byte, sequence length, or reserved bytes.
 func (s *Share) RawData() (rawData []byte, err error) {
@@ -171,6 +194,51 @@ func (s *Share) RawData() (rawData []byte, err error) {
 	return s.data[s.rawDataStartIndex():], nil
 }
 
+// RawDataWithReserved returns the raw share data while taking reserved bytes into account.
+func (s *Share) RawDataUsingReserved() (rawData []byte, err error) {
+	rawDataStartIndexUsingReserved, err := s.rawDataStartIndexUsingReserved()
+	if err != nil {
+		return nil, err
+	}
+
+	// This means share is the last share and does not have any transaction beginning in it
+	if rawDataStartIndexUsingReserved == 0 {
+		return []byte{}, nil
+	}
+	if len(s.data) < rawDataStartIndexUsingReserved {
+		return rawData, fmt.Errorf("share %s is too short to contain raw data", s)
+	}
+
+	return s.data[rawDataStartIndexUsingReserved:], nil
+}
+
+// rawDataStartIndexUsingReserved returns the start index of raw data while accounting for
+// reserved bytes, if it exists in the share.
+func (s *Share) rawDataStartIndexUsingReserved() (int, error) {
+	isStart, err := s.IsSequenceStart()
+	if err != nil {
+		return 0, err
+	}
+	isCompact, err := s.IsCompactShare()
+	if err != nil {
+		return 0, err
+	}
+
+	index := appconsts.NamespaceSize + appconsts.ShareInfoBytes
+	if isStart {
+		index += appconsts.SequenceLenBytes
+	}
+
+	if isCompact {
+		reservedBytes, err := ParseReservedBytes(s.data[index : index+appconsts.CompactShareReservedBytes])
+		if err != nil {
+			return 0, err
+		}
+		return int(reservedBytes), nil
+	}
+	return index, nil
+}
+
 func (s *Share) rawDataStartIndex() int {
 	isStart, err := s.IsSequenceStart()
 	if err != nil {
@@ -180,17 +248,15 @@ func (s *Share) rawDataStartIndex() int {
 	if err != nil {
 		panic(err)
 	}
-	if isStart && isCompact {
-		return appconsts.NamespaceSize + appconsts.ShareInfoBytes + appconsts.SequenceLenBytes + appconsts.CompactShareReservedBytes
-	} else if isStart && !isCompact {
-		return appconsts.NamespaceSize + appconsts.ShareInfoBytes + appconsts.SequenceLenBytes
-	} else if !isStart && isCompact {
-		return appconsts.NamespaceSize + appconsts.ShareInfoBytes + appconsts.CompactShareReservedBytes
-	} else if !isStart && !isCompact {
-		return appconsts.NamespaceSize + appconsts.ShareInfoBytes
-	} else {
-		panic(fmt.Sprintf("unable to determine the rawDataStartIndex for share %s", s.data))
+
+	index := appconsts.NamespaceSize + appconsts.ShareInfoBytes
+	if isStart {
+		index += appconsts.SequenceLenBytes
+	}
+	if isCompact {
+		index += appconsts.CompactShareReservedBytes
 	}
+	return index
 }
 
 func ToBytes(shares []Share) (bytes [][]byte) {

third_party/celestia-app/shares/split_compact_shares.go

@@ -51,6 +51,25 @@ func NewCompactShareSplitter(ns appns.Namespace, shareVersion uint8) *CompactSha
 	}
 }
 
+// NewCompactShareSplitterWithIsCompactFalse returns a CompactShareSplitter using the provided
+// namespace and shareVersion. Also, sets isCompact in the builder to false.
+func NewCompactShareSplitterWithIsCompactFalse(ns appns.Namespace, shareVersion uint8) *CompactShareSplitter {
+	builder := NewBuilder(ns, shareVersion, true)
+	builder.isCompactShare = false
+	sb, err := builder.Init()
+	if err != nil {
+		panic(err)
+	}
+
+	return &CompactShareSplitter{
+		shares:       []Share{},
+		namespace:    ns,
+		shareVersion: shareVersion,
+		shareRanges:  map[coretypes.TxKey]ShareRange{},
+		shareBuilder: sb,
+	}
+}
+
 // WriteTx adds the delimited data for the provided tx to the underlying compact
 // share splitter.
 func (css *CompactShareSplitter) WriteTx(tx coretypes.Tx) error {
@@ -73,6 +92,36 @@ func (css *CompactShareSplitter) WriteTx(tx coretypes.Tx) error {
 	return nil
 }
 
+// write adds the delimited data to the underlying compact shares.
+func (css *CompactShareSplitter) WriteWithNoReservedBytes(rawData []byte) error {
+	if css.done {
+		// remove the last element
+		if !css.shareBuilder.IsEmptyShare() {
+			css.shares = css.shares[:len(css.shares)-1]
+		}
+		css.done = false
+	}
+
+	for {
+		rawDataLeftOver := css.shareBuilder.AddData(rawData)
+		if rawDataLeftOver == nil {
+			break
+		}
+		if err := css.stackPendingWithIsCompactFalse(); err != nil {
+			return err
+		}
+
+		rawData = rawDataLeftOver
+	}
+
+	if css.shareBuilder.AvailableBytes() == 0 {
+		if err := css.stackPendingWithIsCompactFalse(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // write adds the delimited data to the underlying compact shares.
 func (css *CompactShareSplitter) write(rawData []byte) error {
 	if css.done {
@@ -120,6 +169,21 @@ func (css *CompactShareSplitter) stackPending() error {
 	return err
 }
 
+// stackPending will build & add the pending share to accumulated shares
+func (css *CompactShareSplitter) stackPendingWithIsCompactFalse() error {
+	pendingShare, err := css.shareBuilder.Build()
+	if err != nil {
+		return err
+	}
+	css.shares = append(css.shares, *pendingShare)
+
+	// Now we need to create a new builder
+	builder := NewBuilder(css.namespace, css.shareVersion, false)
+	builder.isCompactShare = false
+	css.shareBuilder, err = builder.Init()
+	return err
+}
+
 // Export finalizes and returns the underlying compact shares and a map of
 // shareRanges. All share ranges in the map of shareRanges will be offset (i.e.
 // incremented) by the shareRangeOffset provided. shareRangeOffset should be 0