From 72b640b5ff3b2734e05be29e11086b58c6732739 Mon Sep 17 00:00:00 2001 From: sanaz Date: Thu, 29 Jun 2023 13:52:37 -0700 Subject: [PATCH 01/39] todos --- extendeddatacrossword.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/extendeddatacrossword.go b/extendeddatacrossword.go index c58cf0b..089ef7b 100644 --- a/extendeddatacrossword.go +++ b/extendeddatacrossword.go @@ -44,7 +44,8 @@ type ErrByzantineData struct { } func (e *ErrByzantineData) Error() string { - return fmt.Sprintf("byzantine %s: %d", e.Axis, e.Index) + return fmt.Sprintf( + "byzantine %s: %d", e.Axis, e.Index) } // Repair attempts to repair an incomplete extended data @@ -145,7 +146,7 @@ func (eds *ExtendedDataSquare) solveCrosswordRow( } // Check that rebuilt shares matches appropriate root - err = eds.verifyAgainstRowRoots(rowRoots, uint(r), rebuiltShares, noShareInsertion, nil) + err = eds.verifyAgainstRowRoots(rowRoots, uint(r), rebuiltShares, noShareInsertion, nil) // TODO[?} here is where we should return a new error when attempting t crete the tree root from unordered shares if err != nil { var byzErr *ErrByzantineData if errors.As(err, &byzErr) { @@ -199,7 +200,7 @@ func (eds *ExtendedDataSquare) solveCrosswordCol( } - isExtendedPartIncomplete := !eds.colRangeNoMissingData(uint(c), eds.originalDataWidth, eds.width) + isExtendedPartIncomplete := !eds.colRangeNoMissingData(uint(c), eds.originalDataWidth, eds.width) // TODO[?} why not checking whether half of the shares in the column is available and then attempt to rebuild the missing ones? why just the parity part can be incomplete? // Attempt rebuild rebuiltShares, isDecoded, err := eds.rebuildShares(isExtendedPartIncomplete, shares) if err != nil { @@ -210,11 +211,12 @@ func (eds *ExtendedDataSquare) solveCrosswordCol( } // Check that rebuilt shares matches appropriate root - err = eds.verifyAgainstColRoots(colRoots, uint(c), rebuiltShares, noShareInsertion, nil) + err = eds.verifyAgainstColRoots(colRoots, uint(c), rebuiltShares, noShareInsertion, nil) // TODO[?] we should return a new error here when attempting to create the tree root from unordered shares, hence the ErrByzantineData may need to account for the index of those unordered shares (what if those unordered shares are part of the parity part? it is not clear whether nodes sample from the parity section or not?) + // TODO[?] Then we should in here, retrieve the roots of the opposite cols or rows based on the info enapsulated in the error, and return those data as part of the ErrUnorderedData (maybe or just the old one) if err != nil { var byzErr *ErrByzantineData if errors.As(err, &byzErr) { - byzErr.Shares = shares + byzErr.Shares = shares // it seems these are not the erasure coded shares } return false, false, err } @@ -289,7 +291,7 @@ func (eds *ExtendedDataSquare) verifyAgainstRowRoots( root, err = eds.computeSharesRootWithRebuiltShare(oldShares, Row, r, rebuiltIndex, rebuiltShare) } if err != nil { - return err + return err // TODO[?] depending on the error type it might be the case that the shares were out of order, return the index of the share that was out of order } if !bytes.Equal(root, rowRoots[r]) { @@ -332,6 +334,7 @@ func (eds *ExtendedDataSquare) prerepairSanityCheck( for i := uint(0); i < eds.width; i++ { i := i + // checks if all the shares in the extended data square for this row are present (original and parity) rowIsComplete := noMissingData(eds.row(i), noShareInsertion) colIsComplete := noMissingData(eds.col(i), noShareInsertion) @@ -367,6 +370,7 @@ func (eds *ExtendedDataSquare) prerepairSanityCheck( if rowIsComplete { errs.Go(func() error { + // check if we take the first half of the row and encode it, we get the second half parityShares, err := eds.codec.Encode(eds.rowSlice(i, 0, eds.originalDataWidth)) if err != nil { return err @@ -380,6 +384,7 @@ func (eds *ExtendedDataSquare) prerepairSanityCheck( if colIsComplete { errs.Go(func() error { + // check if we take the first half of the col and encode it, we get the second half parityShares, err := eds.codec.Encode(eds.colSlice(0, i, eds.originalDataWidth)) if err != nil { return err @@ -412,7 +417,7 @@ func (eds *ExtendedDataSquare) computeSharesRoot(shares [][]byte, axis Axis, i u for _, d := range shares { tree.Push(d) } - return tree.Root() + return tree.Root() // TODO[?] depending on the error type it might be the case that the shares were out of order, return the index of the share that was out of order, the index can be encapsulated in the error type } func (eds *ExtendedDataSquare) computeSharesRootWithRebuiltShare(shares [][]byte, axis Axis, i uint, rebuiltIndex int, rebuiltShare []byte) ([]byte, error) { From 48296aa36fc4dafacf2a67effd56165fbb1a696e Mon Sep 17 00:00:00 2001 From: sanaz Date: Mon, 3 Jul 2023 12:46:58 -0700 Subject: [PATCH 02/39] returns ByzantineErrData on root calculation failure or mismatching parity shares --- extendeddatacrossword.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/extendeddatacrossword.go b/extendeddatacrossword.go index bcef5b8..fb006d4 100644 --- a/extendeddatacrossword.go +++ b/extendeddatacrossword.go @@ -311,6 +311,8 @@ func (eds *ExtendedDataSquare) verifyAgainstColRoots( return nil } +// prerepairSanityCheck checks that the roots of the rows and columns, if all the respective constituent shares are available, match the expected roots as supplied in rowRoots and colRoots. +// It also verifies whether the complete rows and columns are correctly erasure coded. func (eds *ExtendedDataSquare) prerepairSanityCheck( rowRoots [][]byte, colRoots [][]byte, @@ -327,10 +329,13 @@ func (eds *ExtendedDataSquare) prerepairSanityCheck( // ensure that the roots are equal rowRoot, err := eds.getRowRoot(i) if err != nil { - return err + // any error regarding the root calculation signifies an issue in the shares e.g., out of order shares + // therefore, it should be treated as byzantine data + return &ErrByzantineData{Row, i, eds.row(i)} } if !bytes.Equal(rowRoots[i], rowRoot) { - return fmt.Errorf("bad root input: row %d expected %v got %v", i, rowRoots[i], rowRoot) + // if the roots are not equal, then the data is byzantine + return &ErrByzantineData{Row, i, eds.row(i)} } return nil }) @@ -353,10 +358,13 @@ func (eds *ExtendedDataSquare) prerepairSanityCheck( // ensure that the roots are equal colRoot, err := eds.getColRoot(i) if err != nil { - return err + // any error regarding the root calculation signifies an issue in the shares e.g., out of order shares + // therefore, it should be treated as byzantine data + return &ErrByzantineData{Col, i, eds.col(i)} } if !bytes.Equal(colRoots[i], colRoot) { - return fmt.Errorf("bad root input: col %d expected %v got %v", i, colRoots[i], colRoot) + // if the roots are not equal, then the data is byzantine + return &ErrByzantineData{Col, i, eds.col(i)} } return nil }) From 0e8ca5041270c2509f7a12c6c3f0ef83c763be51 Mon Sep 17 00:00:00 2001 From: sanaz Date: Mon, 3 Jul 2023 13:04:04 -0700 Subject: [PATCH 03/39] adds godoc for verifyAgainstColRoots and renames oldShares to shares --- extendeddatacrossword.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/extendeddatacrossword.go b/extendeddatacrossword.go index fb006d4..3dd69fd 100644 --- a/extendeddatacrossword.go +++ b/extendeddatacrossword.go @@ -286,26 +286,32 @@ func (eds *ExtendedDataSquare) verifyAgainstRowRoots( return nil } +// verifyAgainstColRoots checks that the shares of column index `c` match their expected column root available in `colRoots`. +// `colRoots` is a slice of the expected roots of the columns of the `eds`. +// `shares` is a slice of the shares of the column index `c` of the `eds`. +// `rebuiltIndex` is the index of the share that was rebuilt, if any. +// `rebuiltShare` is the rebuilt share, if any. func (eds *ExtendedDataSquare) verifyAgainstColRoots( colRoots [][]byte, c uint, - oldShares [][]byte, + shares [][]byte, rebuiltIndex int, rebuiltShare []byte, ) error { var root []byte var err error if rebuiltIndex < 0 || rebuiltShare == nil { - root, err = eds.computeSharesRoot(oldShares, Col, c) + root, err = eds.computeSharesRoot(shares, Col, c) } else { - root, err = eds.computeSharesRootWithRebuiltShare(oldShares, Col, c, rebuiltIndex, rebuiltShare) + root, err = eds.computeSharesRootWithRebuiltShare(shares, Col, c, rebuiltIndex, rebuiltShare) } if err != nil { - return err + // the shares are set to nil, as the caller will populate them + return &ErrByzantineData{Col, c, nil} } if !bytes.Equal(root, colRoots[c]) { - return &ErrByzantineData{Col, c, nil} + return &ErrByzantineData{Col, c, nil} // the shares are set to nil, as the caller will populate them } return nil @@ -397,6 +403,7 @@ func noMissingData(input [][]byte, rebuiltIndex int) bool { return true } +// computeSharesRoot computes the root of the shares of the specified axis (column or row) and index `i`. func (eds *ExtendedDataSquare) computeSharesRoot(shares [][]byte, axis Axis, i uint) ([]byte, error) { tree := eds.createTreeFn(axis, i) for _, d := range shares { @@ -405,6 +412,7 @@ func (eds *ExtendedDataSquare) computeSharesRoot(shares [][]byte, axis Axis, i u return tree.Root() // TODO[?] depending on the error type it might be the case that the shares were out of order, return the index of the share that was out of order, the index can be encapsulated in the error type } +// computeSharesRootWithRebuiltShare computes the root of the shares with the rebuilt share `rebuiltShare` at the specified index `rebuiltIndex`. func (eds *ExtendedDataSquare) computeSharesRootWithRebuiltShare(shares [][]byte, axis Axis, i uint, rebuiltIndex int, rebuiltShare []byte) ([]byte, error) { tree := eds.createTreeFn(axis, i) for _, d := range shares[:rebuiltIndex] { From cc82d3f3d244b5d661e8ed4c054ad6c714a9a666 Mon Sep 17 00:00:00 2001 From: sanaz Date: Mon, 3 Jul 2023 13:59:46 -0700 Subject: [PATCH 04/39] return ErrByzantineData when the root verification of the orthogonal axis fails --- extendeddatacrossword.go | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/extendeddatacrossword.go b/extendeddatacrossword.go index 3dd69fd..a9e1a01 100644 --- a/extendeddatacrossword.go +++ b/extendeddatacrossword.go @@ -137,7 +137,7 @@ func (eds *ExtendedDataSquare) solveCrosswordRow( shares[c] = vectorData[c] } - // Attempt rebuild + // Attempt rebuild the row rebuiltShares, isDecoded, err := eds.rebuildShares(shares) if err != nil { return false, false, err @@ -147,7 +147,7 @@ func (eds *ExtendedDataSquare) solveCrosswordRow( } // Check that rebuilt shares matches appropriate root - err = eds.verifyAgainstRowRoots(rowRoots, uint(r), rebuiltShares, noShareInsertion, nil) // TODO[?} here is where we should return a new error when attempting t crete the tree root from unordered shares + err = eds.verifyAgainstRowRoots(rowRoots, uint(r), rebuiltShares, noShareInsertion, nil) if err != nil { var byzErr *ErrByzantineData if errors.As(err, &byzErr) { @@ -162,9 +162,17 @@ func (eds *ExtendedDataSquare) solveCrosswordRow( if col[r] != nil { continue // not newly completed } - if noMissingData(col, r) { // not completed - err := eds.verifyAgainstColRoots(colRoots, uint(c), col, r, rebuiltShares[c]) + if noMissingData(col, r) { // completed TODO[?] or half completed? + err := eds.verifyAgainstColRoots(colRoots, uint(c), col, r, rebuiltShares[r]) if err != nil { + var byzErr *ErrByzantineData + if errors.As(err, &byzErr) { + // make a copy of the column shares + colShares := make([][]byte, eds.width) + copy(colShares, col) + colShares[r] = col[r] // include the newly completed share + byzErr.Shares = colShares + } return false, false, err } } @@ -216,7 +224,7 @@ func (eds *ExtendedDataSquare) solveCrosswordCol( if err != nil { var byzErr *ErrByzantineData if errors.As(err, &byzErr) { - byzErr.Shares = shares // it seems these are not the erasure coded shares + byzErr.Shares = shares } return false, false, err } @@ -228,8 +236,16 @@ func (eds *ExtendedDataSquare) solveCrosswordCol( continue // not newly completed } if noMissingData(row, c) { // not completed - err := eds.verifyAgainstRowRoots(rowRoots, uint(r), row, c, rebuiltShares[r]) + err := eds.verifyAgainstRowRoots(rowRoots, uint(r), row, c, rebuiltShares[c]) if err != nil { + var byzErr *ErrByzantineData + if errors.As(err, &byzErr) { + // make a copy of the column shares + rowShares := make([][]byte, eds.width) + copy(rowShares, eds.row(uint(r))) + rowShares[r] = rebuiltShares[r] // include the newly completed share + byzErr.Shares = rowShares + } return false, false, err } } @@ -276,10 +292,13 @@ func (eds *ExtendedDataSquare) verifyAgainstRowRoots( root, err = eds.computeSharesRootWithRebuiltShare(oldShares, Row, r, rebuiltIndex, rebuiltShare) } if err != nil { - return err // TODO[?] depending on the error type it might be the case that the shares were out of order, return the index of the share that was out of order + // any error during the computation of the root is considered byzantine + // the shares are set to nil, as the caller will populate them + return &ErrByzantineData{Row, r, nil} } if !bytes.Equal(root, rowRoots[r]) { + // the shares are set to nil, as the caller will populate them return &ErrByzantineData{Row, r, nil} } @@ -291,6 +310,7 @@ func (eds *ExtendedDataSquare) verifyAgainstRowRoots( // `shares` is a slice of the shares of the column index `c` of the `eds`. // `rebuiltIndex` is the index of the share that was rebuilt, if any. // `rebuiltShare` is the rebuilt share, if any. +// Returns a ErrByzantineData error if the computed root does not match the expected root or if the root computation fails. func (eds *ExtendedDataSquare) verifyAgainstColRoots( colRoots [][]byte, c uint, @@ -311,7 +331,8 @@ func (eds *ExtendedDataSquare) verifyAgainstColRoots( } if !bytes.Equal(root, colRoots[c]) { - return &ErrByzantineData{Col, c, nil} // the shares are set to nil, as the caller will populate them + // the shares are set to nil, as the caller will populate them + return &ErrByzantineData{Col, c, nil} } return nil From 6b772713c3d716708ca49f7a785df15f95d0ea7d Mon Sep 17 00:00:00 2001 From: sanaz Date: Mon, 3 Jul 2023 14:17:28 -0700 Subject: [PATCH 05/39] cleans up the docs and comments --- extendeddatacrossword.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/extendeddatacrossword.go b/extendeddatacrossword.go index a9e1a01..5746106 100644 --- a/extendeddatacrossword.go +++ b/extendeddatacrossword.go @@ -162,7 +162,7 @@ func (eds *ExtendedDataSquare) solveCrosswordRow( if col[r] != nil { continue // not newly completed } - if noMissingData(col, r) { // completed TODO[?] or half completed? + if noMissingData(col, r) { // completed err := eds.verifyAgainstColRoots(colRoots, uint(c), col, r, rebuiltShares[r]) if err != nil { var byzErr *ErrByzantineData @@ -219,8 +219,7 @@ func (eds *ExtendedDataSquare) solveCrosswordCol( } // Check that rebuilt shares matches appropriate root - err = eds.verifyAgainstColRoots(colRoots, uint(c), rebuiltShares, noShareInsertion, nil) // TODO[?] we should return a new error here when attempting to create the tree root from unordered shares, hence the ErrByzantineData may need to account for the index of those unordered shares (what if those unordered shares are part of the parity part? it is not clear whether nodes sample from the parity section or not?) - // TODO[?] Then we should in here, retrieve the roots of the opposite cols or rows based on the info enapsulated in the error, and return those data as part of the ErrUnorderedData (maybe or just the old one) + err = eds.verifyAgainstColRoots(colRoots, uint(c), rebuiltShares, noShareInsertion, nil) if err != nil { var byzErr *ErrByzantineData if errors.As(err, &byzErr) { @@ -430,7 +429,7 @@ func (eds *ExtendedDataSquare) computeSharesRoot(shares [][]byte, axis Axis, i u for _, d := range shares { tree.Push(d) } - return tree.Root() // TODO[?] depending on the error type it might be the case that the shares were out of order, return the index of the share that was out of order, the index can be encapsulated in the error type + return tree.Root() } // computeSharesRootWithRebuiltShare computes the root of the shares with the rebuilt share `rebuiltShare` at the specified index `rebuiltIndex`. From 9271b3355305e0d1cb23e028fe00ddb140d32821 Mon Sep 17 00:00:00 2001 From: sanaz Date: Mon, 3 Jul 2023 14:25:08 -0700 Subject: [PATCH 06/39] fixes a typo --- extendeddatacrossword.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extendeddatacrossword.go b/extendeddatacrossword.go index 5746106..2f51c7d 100644 --- a/extendeddatacrossword.go +++ b/extendeddatacrossword.go @@ -163,7 +163,7 @@ func (eds *ExtendedDataSquare) solveCrosswordRow( continue // not newly completed } if noMissingData(col, r) { // completed - err := eds.verifyAgainstColRoots(colRoots, uint(c), col, r, rebuiltShares[r]) + err := eds.verifyAgainstColRoots(colRoots, uint(c), col, r, rebuiltShares[c]) if err != nil { var byzErr *ErrByzantineData if errors.As(err, &byzErr) { From 9ed797dd43ffbee9e5212d769f0330419093a123 Mon Sep 17 00:00:00 2001 From: sanaz Date: Mon, 3 Jul 2023 14:41:34 -0700 Subject: [PATCH 07/39] fixes a wrong index, and updates the orthogonal axis prior to returning the error --- extendeddatacrossword.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/extendeddatacrossword.go b/extendeddatacrossword.go index 2f51c7d..8cf7c3a 100644 --- a/extendeddatacrossword.go +++ b/extendeddatacrossword.go @@ -162,15 +162,15 @@ func (eds *ExtendedDataSquare) solveCrosswordRow( if col[r] != nil { continue // not newly completed } - if noMissingData(col, r) { // completed + eds.setCell(uint(r), uint(c), rebuiltShares[c]) // include the newly completed share + if noMissingData(col, r) { // completed err := eds.verifyAgainstColRoots(colRoots, uint(c), col, r, rebuiltShares[c]) if err != nil { var byzErr *ErrByzantineData if errors.As(err, &byzErr) { // make a copy of the column shares colShares := make([][]byte, eds.width) - copy(colShares, col) - colShares[r] = col[r] // include the newly completed share + copy(colShares, eds.col(uint(c))) byzErr.Shares = colShares } return false, false, err @@ -234,15 +234,15 @@ func (eds *ExtendedDataSquare) solveCrosswordCol( if row[c] != nil { continue // not newly completed } - if noMissingData(row, c) { // not completed - err := eds.verifyAgainstRowRoots(rowRoots, uint(r), row, c, rebuiltShares[c]) + eds.setCell(uint(r), uint(c), rebuiltShares[r]) // include the newly completed share + if noMissingData(row, c) { // completed + err := eds.verifyAgainstRowRoots(rowRoots, uint(r), row, c, rebuiltShares[r]) if err != nil { var byzErr *ErrByzantineData if errors.As(err, &byzErr) { // make a copy of the column shares rowShares := make([][]byte, eds.width) copy(rowShares, eds.row(uint(r))) - rowShares[r] = rebuiltShares[r] // include the newly completed share byzErr.Shares = rowShares } return false, false, err From ec150291b2315dd9febf86fc75e685230a021fab Mon Sep 17 00:00:00 2001 From: sanaz Date: Mon, 3 Jul 2023 14:56:29 -0700 Subject: [PATCH 08/39] removes redundant copy of rebuilt shares --- extendeddatacrossword.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/extendeddatacrossword.go b/extendeddatacrossword.go index 8cf7c3a..ce66647 100644 --- a/extendeddatacrossword.go +++ b/extendeddatacrossword.go @@ -179,9 +179,9 @@ func (eds *ExtendedDataSquare) solveCrosswordRow( } // Insert rebuilt shares into square. - for c, s := range rebuiltShares { - eds.setCell(uint(r), uint(c), s) - } + // for c, s := range rebuiltShares { + // eds.setCell(uint(r), uint(c), s) + // } return true, true, nil } @@ -251,9 +251,9 @@ func (eds *ExtendedDataSquare) solveCrosswordCol( } // Insert rebuilt shares into square. - for r, s := range rebuiltShares { - eds.setCell(uint(r), uint(c), s) - } + // for r, s := range rebuiltShares { + // eds.setCell(uint(r), uint(c), s) + // } return true, true, nil } From 6caeb28a99f2199c44c6eaf88fd5f5f1bad189fb Mon Sep 17 00:00:00 2001 From: sanaz Date: Tue, 4 Jul 2023 13:50:38 -0700 Subject: [PATCH 09/39] reverts setting the missing share --- extendeddatacrossword.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/extendeddatacrossword.go b/extendeddatacrossword.go index ce66647..17b5a3d 100644 --- a/extendeddatacrossword.go +++ b/extendeddatacrossword.go @@ -5,7 +5,6 @@ import ( "context" "errors" "fmt" - "golang.org/x/sync/errgroup" ) @@ -162,8 +161,7 @@ func (eds *ExtendedDataSquare) solveCrosswordRow( if col[r] != nil { continue // not newly completed } - eds.setCell(uint(r), uint(c), rebuiltShares[c]) // include the newly completed share - if noMissingData(col, r) { // completed + if noMissingData(col, r) { // completed err := eds.verifyAgainstColRoots(colRoots, uint(c), col, r, rebuiltShares[c]) if err != nil { var byzErr *ErrByzantineData @@ -179,9 +177,9 @@ func (eds *ExtendedDataSquare) solveCrosswordRow( } // Insert rebuilt shares into square. - // for c, s := range rebuiltShares { - // eds.setCell(uint(r), uint(c), s) - // } + for c, s := range rebuiltShares { + eds.setCell(uint(r), uint(c), s) + } return true, true, nil } @@ -234,8 +232,7 @@ func (eds *ExtendedDataSquare) solveCrosswordCol( if row[c] != nil { continue // not newly completed } - eds.setCell(uint(r), uint(c), rebuiltShares[r]) // include the newly completed share - if noMissingData(row, c) { // completed + if noMissingData(row, c) { // completed err := eds.verifyAgainstRowRoots(rowRoots, uint(r), row, c, rebuiltShares[r]) if err != nil { var byzErr *ErrByzantineData @@ -251,9 +248,9 @@ func (eds *ExtendedDataSquare) solveCrosswordCol( } // Insert rebuilt shares into square. - // for r, s := range rebuiltShares { - // eds.setCell(uint(r), uint(c), s) - // } + for r, s := range rebuiltShares { + eds.setCell(uint(r), uint(c), s) + } return true, true, nil } From 12a449cf821321efa61660a58e675befffdf9ba3 Mon Sep 17 00:00:00 2001 From: sanaz Date: Tue, 4 Jul 2023 15:26:40 -0700 Subject: [PATCH 10/39] use the already copied shares --- extendeddatacrossword.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/extendeddatacrossword.go b/extendeddatacrossword.go index b0df7cd..061df0a 100644 --- a/extendeddatacrossword.go +++ b/extendeddatacrossword.go @@ -173,10 +173,7 @@ func (eds *ExtendedDataSquare) solveCrosswordRow( if err != nil { var byzErr *ErrByzantineData if errors.As(err, &byzErr) { - // make a copy of the column shares - colShares := make([][]byte, eds.width) - copy(colShares, eds.col(uint(c))) - byzErr.Shares = colShares + byzErr.Shares = shares } return false, false, err } @@ -249,10 +246,7 @@ func (eds *ExtendedDataSquare) solveCrosswordCol( if err != nil { var byzErr *ErrByzantineData if errors.As(err, &byzErr) { - // make a copy of the column shares - rowShares := make([][]byte, eds.width) - copy(rowShares, eds.row(uint(r))) - byzErr.Shares = rowShares + byzErr.Shares = shares } return false, false, err } From 72c551076d49992b5e0236ccc579735af8ebd4c1 Mon Sep 17 00:00:00 2001 From: sanaz Date: Wed, 5 Jul 2023 10:20:16 -0700 Subject: [PATCH 11/39] initial tests --- extendeddatacrossword_test.go | 82 +++++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index 9524a80..81e2c2f 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -4,11 +4,11 @@ import ( "bytes" "errors" "fmt" - "math/rand" - "testing" - + "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "math/rand" + "testing" ) // PseudoFraudProof is an example fraud proof. @@ -260,6 +260,65 @@ func TestCorruptedEdsReturnsErrByzantineData(t *testing.T) { } } +func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { + shareSize := 64 + corruptChunk := bytes.Repeat([]byte{66}, shareSize) + + tests := []struct { + name string + coords [][]uint + values [][]byte + }{ + { + name: "corrupt a chunk in the original data square", + coords: [][]uint{{0, 0}}, + values: [][]byte{corruptChunk}, + }, + { + name: "corrupt a chunk in the extended data square", + coords: [][]uint{{0, 3}}, + values: [][]byte{corruptChunk}, + }, + { + name: "corrupt a chunk at (0, 0) and delete shares from the rest of the row", + coords: [][]uint{{0, 0}, {0, 1}, {0, 2}, {0, 3}}, + values: [][]byte{corruptChunk, nil, nil, nil}, + }, + { + name: "corrupt a chunk at (3, 0) and delete part of the first row ", + coords: [][]uint{{3, 0}, {0, 1}, {0, 2}, {0, 3}}, + values: [][]byte{corruptChunk, nil, nil, nil}, + }, + } + + for codecName, codec := range codecs { + t.Run(codecName, func(t *testing.T) { + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + eds := createTestEds(codec, shareSize) + for i, coords := range test.coords { + x := coords[0] + y := coords[1] + eds.setCell(x, y, test.values[i]) + } + rowRoots, err := eds.getRowRoots() + assert.NoError(t, err) + + colRoots, err := eds.getColRoots() + assert.NoError(t, err) + + err = eds.Repair(rowRoots, colRoots) + assert.Error(t, err) + + // due to parallelisation, the ErrByzantineData axis may be either row or col + var byzData *ErrByzantineData + assert.ErrorAs(t, err, &byzData, "did not return a ErrByzantineData for a bad col or row") + }) + } + }) + } +} + func BenchmarkRepair(b *testing.B) { // For different ODS sizes for originalDataWidth := 4; originalDataWidth <= 512; originalDataWidth *= 2 { @@ -343,3 +402,20 @@ func createTestEds(codec Codec, shareSize int) *ExtendedDataSquare { return eds } + +func createTestEdsWithNMT(codec Codec, shareSize int) *ExtendedDataSquare { + ones := bytes.Repeat([]byte{1}, shareSize) + twos := bytes.Repeat([]byte{2}, shareSize) + threes := bytes.Repeat([]byte{3}, shareSize) + fours := bytes.Repeat([]byte{4}, shareSize) + + eds, err := ComputeExtendedDataSquare([][]byte{ + ones, twos, + threes, fours, + }, codec, wrapper.NewConstructor(uint64(shareSize))) + if err != nil { + panic(err) + } + + return eds +} From 113f1c3830005987cf2c032e3e90ff5bf3573697 Mon Sep 17 00:00:00 2001 From: sanaz Date: Wed, 5 Jul 2023 10:58:55 -0700 Subject: [PATCH 12/39] resolves nmt wrapper dependency --- go.mod | 30 ++++++--- go.sum | 202 ++++++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 199 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 747ca8c..986875e 100644 --- a/go.mod +++ b/go.mod @@ -3,24 +3,36 @@ module github.com/celestiaorg/rsmt2d go 1.20 require ( - github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 - github.com/stretchr/testify v1.7.0 + github.com/celestiaorg/celestia-app v1.0.0-rc7 + github.com/celestiaorg/merkletree v0.0.0-20230308153949-c33506a7aa26 + github.com/stretchr/testify v1.8.4 ) require ( - github.com/klauspost/reedsolomon v1.11.1 - golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 + github.com/klauspost/reedsolomon v1.11.8 + golang.org/x/sync v0.3.0 +) + +require ( + github.com/celestiaorg/nmt v0.17.0 // indirect + github.com/petermattis/goid v0.0.0-20230518223814-80aa455d8761 // indirect + github.com/sasha-s/go-deadlock v0.3.1 // indirect + github.com/tendermint/tendermint v0.35.9 // indirect ) require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/klauspost/cpuid/v2 v2.1.1 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/text v0.2.0 // indirect - github.com/minio/sha256-simd v1.0.0 + github.com/minio/sha256-simd v1.0.1 github.com/pmezard/go-difflib v1.0.0 // indirect gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975 // indirect - golang.org/x/crypto v0.1.0 // indirect - golang.org/x/sys v0.1.0 // indirect + golang.org/x/sys v0.10.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace ( + github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 + github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.23.0-tm-v0.34.28 ) diff --git a/go.sum b/go.sum index bb92e06..82839b2 100644 --- a/go.sum +++ b/go.sum @@ -1,48 +1,202 @@ -github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= -github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= +cosmossdk.io/errors v1.0.0-beta.7 h1:gypHW76pTQGVnHKo6QBkb4yFOJjC+sUGRc5Al3Odj1w= +cosmossdk.io/math v1.0.0-rc.0 h1:ml46ukocrAAoBpYKMidF0R2tQJ1Uxfns0yH8wqgMAFc= +filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmGfU= +github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= +github.com/99designs/keyring v1.2.1 h1:tYLp1ULvO7i3fI5vE21ReQuj99QFSs7lGm0xWyJo87o= +github.com/ChainSafe/go-schnorrkel v1.0.0 h1:3aDA67lAykLaG1y3AOjs88dMxC88PgUuHRrLeDnvGIM= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/aws/aws-sdk-go v1.40.45 h1:QN1nsY27ssD/JmW4s83qmSb+uL6DG4GmCDzjmJB4xUI= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= +github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= +github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= +github.com/celestiaorg/celestia-app v1.0.0-rc7 h1:xkLEHrC/vjgW6qzsoqrt2CP6Q9c8fbtn3CGLFIqe5UE= +github.com/celestiaorg/celestia-app v1.0.0-rc7/go.mod h1:X0R6s+LvfusZu+jBj/2SbTm4Nb/H1R2MD1CnR4fwQno= +github.com/celestiaorg/celestia-core v1.23.0-tm-v0.34.28 h1:G7/rq6xTnuFf3XsVZEcl/Sa6vtagm9NQNhaUaSgjvy0= +github.com/celestiaorg/celestia-core v1.23.0-tm-v0.34.28/go.mod h1:J/GsBjoTZaFz71VeyrLZbG8rV+Rzi6oFEUZUipQ97hQ= +github.com/celestiaorg/merkletree v0.0.0-20230308153949-c33506a7aa26 h1:P2RI1xJ49EZ8cuHMcH+ZSBonfRDtBS8OS9Jdt1BWX3k= +github.com/celestiaorg/merkletree v0.0.0-20230308153949-c33506a7aa26/go.mod h1:2m8ukndOegwB0PU0AfJCwDUQHqd7QQRlSXvQL5VToVY= +github.com/celestiaorg/nmt v0.17.0 h1:/k8YLwJvuHgT/jQ435zXKaDX811+sYEMXL4B/vYdSLU= +github.com/celestiaorg/nmt v0.17.0/go.mod h1:ZndCeAR4l9lxm7W51ouoyTo1cxhtFgK+4DpEIkxRA3A= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= +github.com/cockroachdb/apd/v2 v2.0.2 h1:weh8u7Cneje73dDh+2tEVLUvyBc89iwepWCD8b8034E= +github.com/confio/ics23/go v0.9.0 h1:cWs+wdbS2KRPZezoaaj+qBleXgUk5WOQFMP3CQFGTr4= +github.com/cosmos/btcutil v1.0.5 h1:t+ZFcX77LpKtDBhjucvnOH8C2l2ioGsBNEQ3jef8xFk= +github.com/cosmos/cosmos-proto v1.0.0-alpha8 h1:d3pCRuMYYvGA5bM0ZbbjKn+AoQD4A7dyNG2wzwWalUw= +github.com/cosmos/cosmos-sdk v0.46.13 h1:LhL6WDBadczqBuCW0t5BHUzGQR3vbujdOYOfU0ORt+o= +github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY= +github.com/cosmos/gorocksdb v1.2.0 h1:d0l3jJG8M4hBouIZq0mDUHZ+zjOx044J3nGRskwTb4Y= +github.com/cosmos/iavl v0.19.6 h1:XY78yEeNPrEYyNCKlqr9chrwoeSDJ0bV2VjocTk//OU= +github.com/cosmos/ledger-cosmos-go v0.12.2 h1:/XYaBlE2BJxtvpkHiBm97gFGSGmYGKunKyF3nNqAXZA= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0= -github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/klauspost/reedsolomon v1.11.1 h1:0gCWQXOB8pVe1Y5SGozDA5t2qoVxX3prsV+qHgI/Fik= -github.com/klauspost/reedsolomon v1.11.1/go.mod h1:FXLZzlJIdfqEnQLdUKWNRuMZg747hZ4oYp2Ml60Lb/k= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= +github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= +github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac h1:opbrjaN/L8gg6Xh5D04Tem+8xVcz6ajZlGCs49mQgyg= +github.com/dvsekhvalnov/jose2go v1.5.0 h1:3j8ya4Z4kMCwT5nXIKFSV84YS+HdqSSO0VsTQxaLAeM= +github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/go-kit/kit v0.12.0 h1:e4o3o3IsBfAKQh5Qbbiqyfu97Ku7jrO/JbohvztANh4= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= +github.com/gogo/gateway v1.1.0 h1:u0SuhL9+Il+UbjM9VIE3ntfRujKbvVpFvNB4HbjeVQ0= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= +github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= +github.com/gtank/merlin v0.1.1 h1:eQ90iG7K9pOhtereWsmyRJ6RAwcP4tHTDBHXNg+u5is= +github.com/gtank/ristretto255 v0.1.2 h1:JEqUCPA1NvLq5DwYtuzigd7ss8fwbYay9fi4/5uMzcc= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-getter v1.6.1 h1:NASsgP4q6tL94WH6nJxKWj8As2H/2kop/bB1d8JMyRY= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 h1:aSVUgRRRtOrZOC1fYmY9gV0e9z/Iu+xNVSASWjsuyGU= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= +github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= +github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= +github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= +github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 h1:QRUSJEgZn2Snx0EmT/QLXibWjSUDjKWvXIT19NBVp94= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us= +github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= +github.com/petermattis/goid v0.0.0-20230518223814-80aa455d8761 h1:W04oB3d0J01W5jgYRGKsV8LCM6g9EkCvPkZcmFuy0OE= +github.com/petermattis/goid v0.0.0-20230518223814-80aa455d8761/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= +github.com/regen-network/protobuf v1.3.3-alpha.regen.1 h1:OHEc+q5iIAXpqiqFKeLpu5NwTIkVXUs48vFMwzqpqY4= +github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0= +github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM= +github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= +github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= +github.com/tendermint/go-amino v0.16.0 h1:GyhmgQKvqF82e2oZeuMSp9JTN0N09emoSZlb2lyGa2E= +github.com/tendermint/tm-db v0.6.7 h1:fE00Cbl0jayAoqlExN6oyQJ7fR/ZtoVOmvPJ//+shu8= +github.com/tidwall/btree v1.5.0 h1:iV0yVY/frd7r6qGBXfEYs7DH0gTDgrKTrDjS7xt/IyQ= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zondax/hid v0.9.1 h1:gQe66rtmyZ8VeGFcOpbuH3r7erYtNEAezCAYu8LdkJo= +github.com/zondax/ledger-go v0.14.1 h1:Pip65OOl4iJ84WTpA4BKChvOufMhhbxED3BaihoZN4c= gitlab.com/NebulousLabs/errors v0.0.0-20171229012116-7ead97ef90b8/go.mod h1:ZkMZ0dpQyWwlENaeZVBiQRjhMEZvk6VTXquzl3FOFP8= gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975 h1:L/ENs/Ar1bFzUeKx6m3XjlmBgIUlykX9dzvp5k9NGxc= gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975/go.mod h1:ZkMZ0dpQyWwlENaeZVBiQRjhMEZvk6VTXquzl3FOFP8= gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40 h1:dizWJqTWjwyD8KGcMOwgrkqu1JIkofYgKkmDeNE7oAs= gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40/go.mod h1:rOnSnoRyxMI3fe/7KIbVcsHRGxe30OONv8dEgo+vCfA= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200109152110-61a87790db17/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 h1:cu5kTvlzcw1Q5S9f5ip1/cpiB4nXvw1XYzFPGgzLUOY= -golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0 h1:LGJsf5LRplCck6jUCH3dBL2dmycNruWNF5xugkSlfXw= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= From b75d1f2b4e3fd16ceb945c3e00996e517fbc2116 Mon Sep 17 00:00:00 2001 From: sanaz Date: Wed, 5 Jul 2023 12:23:58 -0700 Subject: [PATCH 13/39] uses NMT for the unordered shares test --- extendeddatacrossword_test.go | 52 ++++++++++++++--------------------- 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index 81e2c2f..d8c7291 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -262,7 +262,7 @@ func TestCorruptedEdsReturnsErrByzantineData(t *testing.T) { func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { shareSize := 64 - corruptChunk := bytes.Repeat([]byte{66}, shareSize) + // corruptChunk := bytes.Repeat([]byte{66}, shareSize) tests := []struct { name string @@ -270,24 +270,9 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { values [][]byte }{ { - name: "corrupt a chunk in the original data square", - coords: [][]uint{{0, 0}}, - values: [][]byte{corruptChunk}, - }, - { - name: "corrupt a chunk in the extended data square", - coords: [][]uint{{0, 3}}, - values: [][]byte{corruptChunk}, - }, - { - name: "corrupt a chunk at (0, 0) and delete shares from the rest of the row", - coords: [][]uint{{0, 0}, {0, 1}, {0, 2}, {0, 3}}, - values: [][]byte{corruptChunk, nil, nil, nil}, - }, - { - name: "corrupt a chunk at (3, 0) and delete part of the first row ", - coords: [][]uint{{3, 0}, {0, 1}, {0, 2}, {0, 3}}, - values: [][]byte{corruptChunk, nil, nil, nil}, + name: "no corruption", + coords: [][]uint{}, + values: [][]byte{}, }, } @@ -295,24 +280,22 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { t.Run(codecName, func(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - eds := createTestEds(codec, shareSize) - for i, coords := range test.coords { - x := coords[0] - y := coords[1] - eds.setCell(x, y, test.values[i]) - } + eds := createTestEdsWithNMT(codec, shareSize) rowRoots, err := eds.getRowRoots() assert.NoError(t, err) colRoots, err := eds.getColRoots() assert.NoError(t, err) + for i, coords := range test.coords { + x := coords[0] + y := coords[1] + eds.setCell(x, y, test.values[i]) + } + err = eds.Repair(rowRoots, colRoots) - assert.Error(t, err) + assert.NoError(t, err) - // due to parallelisation, the ErrByzantineData axis may be either row or col - var byzData *ErrByzantineData - assert.ErrorAs(t, err, &byzData, "did not return a ErrByzantineData for a bad col or row") }) } }) @@ -408,11 +391,16 @@ func createTestEdsWithNMT(codec Codec, shareSize int) *ExtendedDataSquare { twos := bytes.Repeat([]byte{2}, shareSize) threes := bytes.Repeat([]byte{3}, shareSize) fours := bytes.Repeat([]byte{4}, shareSize) + edsWidth := 4 // number of shares per row/column in the extended data square + odsWidth := edsWidth / 2 // number of shares per row/column in the original data square eds, err := ComputeExtendedDataSquare([][]byte{ - ones, twos, - threes, fours, - }, codec, wrapper.NewConstructor(uint64(shareSize))) + ones, twos, threes, fours, + // original data square would be like + // row1: 1 2 + // row2: 3 4 + }, codec, wrapper.NewConstructor(uint64(odsWidth))) + if err != nil { panic(err) } From a523a65ae4d3bcdd950297388dbd97375d0a7464 Mon Sep 17 00:00:00 2001 From: sanaz Date: Thu, 6 Jul 2023 12:38:48 -0700 Subject: [PATCH 14/39] adds the wrapper package with implementation --- go.mod | 5 +- go.sum | 8 +++ nmtwrapper.go | 135 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 nmtwrapper.go diff --git a/go.mod b/go.mod index 747ca8c..a12ffda 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,8 @@ go 1.20 require ( github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 - github.com/stretchr/testify v1.7.0 + github.com/celestiaorg/nmt v0.17.0 + github.com/stretchr/testify v1.8.1 ) require ( @@ -22,5 +23,5 @@ require ( golang.org/x/crypto v0.1.0 // indirect golang.org/x/sys v0.1.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index bb92e06..6c6e2b8 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= +github.com/celestiaorg/nmt v0.17.0 h1:/k8YLwJvuHgT/jQ435zXKaDX811+sYEMXL4B/vYdSLU= +github.com/celestiaorg/nmt v0.17.0/go.mod h1:ZndCeAR4l9lxm7W51ouoyTo1cxhtFgK+4DpEIkxRA3A= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -20,8 +22,13 @@ github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= gitlab.com/NebulousLabs/errors v0.0.0-20171229012116-7ead97ef90b8/go.mod h1:ZkMZ0dpQyWwlENaeZVBiQRjhMEZvk6VTXquzl3FOFP8= gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975 h1:L/ENs/Ar1bFzUeKx6m3XjlmBgIUlykX9dzvp5k9NGxc= gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975/go.mod h1:ZkMZ0dpQyWwlENaeZVBiQRjhMEZvk6VTXquzl3FOFP8= @@ -46,3 +53,4 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/nmtwrapper.go b/nmtwrapper.go new file mode 100644 index 0000000..237138f --- /dev/null +++ b/nmtwrapper.go @@ -0,0 +1,135 @@ +package rsmt2d + +import ( + "fmt" + "github.com/celestiaorg/nmt" + "github.com/celestiaorg/nmt/namespace" + "github.com/minio/sha256-simd" +) + +// Fulfills the Tree interface and TreeConstructorFn function +var ( + _ TreeConstructorFn = NewConstructor(0) + _ Tree = &ErasuredNamespacedMerkleTree{} +) + +const NamespaceSize = 32 + +var ParitySharesNamespaceBytes = []byte{1} + +// ErasuredNamespacedMerkleTree wraps NamespaceMerkleTree to conform to the +// rsmt2d.Tree interface while also providing the correct namespaces to the +// underlying NamespaceMerkleTree. It does this by adding the already included +// namespace to the first half of the tree, and then uses the parity namespace +// ID for each share pushed to the second half of the tree. This allows for the +// namespaces to be included in the erasure data, while also keeping the nmt +// library sufficiently general +type ErasuredNamespacedMerkleTree struct { + squareSize uint64 // note: this refers to the width of the original square before erasure-coded + options []nmt.Option + tree NMTTree + // axisIndex is the index of the axis (row or column) that this tree is on. This is passed + // by rsmt2d and used to help determine which quadrant each leaf belongs to. + axisIndex uint64 + // shareIndex is the index of the share in a row or column that is being + // pushed to the tree. It is expected to be in the range: 0 <= shareIndex < + // 2*squareSize. shareIndex is used to help determine which quadrant each + // leaf belongs to, along with keeping track of how many leaves have been + // added to the tree so far. + shareIndex uint64 +} + +// Tree is an interface that wraps the methods of the underlying +// NamespaceMerkleTree that are used by ErasuredNamespacedMerkleTree. This +// interface is mainly used for testing. It is not recommended to use this +// interface by implementing a different implementation. +type NMTTree interface { + Root() ([]byte, error) + Push(namespacedData namespace.PrefixedData) error +} + +// NewErasuredNamespacedMerkleTree creates a new ErasuredNamespacedMerkleTree +// with an underlying NMT of namespace size `NamespaceSize` and with +// `ignoreMaxNamespace=true`. axisIndex is the index of the row or column that +// this tree is committing to. squareSize must be greater than zero. +func NewErasuredNamespacedMerkleTree(squareSize uint64, axisIndex uint, options ...nmt.Option) ErasuredNamespacedMerkleTree { + if squareSize == 0 { + panic("cannot create a ErasuredNamespacedMerkleTree of squareSize == 0") + } + options = append(options, nmt.NamespaceIDSize(NamespaceSize)) + options = append(options, nmt.IgnoreMaxNamespace(true)) + tree := nmt.New(sha256.New(), options...) + return ErasuredNamespacedMerkleTree{squareSize: squareSize, options: options, tree: tree, axisIndex: uint64(axisIndex), shareIndex: 0} +} + +type constructor struct { + squareSize uint64 + opts []nmt.Option +} + +// NewConstructor creates a tree constructor function as required by rsmt2d to +// calculate the data root. It creates that tree using the +// wrapper.ErasuredNamespacedMerkleTree. +func NewConstructor(squareSize uint64, opts ...nmt.Option) TreeConstructorFn { + return constructor{ + squareSize: squareSize, + opts: opts, + }.NewTree +} + +// NewTree creates a new Tree using the +// ErasuredNamespacedMerkleTree with predefined square size and +// nmt.Options +func (c constructor) NewTree(_ Axis, axisIndex uint) Tree { + newTree := NewErasuredNamespacedMerkleTree(c.squareSize, axisIndex, c.opts...) + return &newTree +} + +// Push adds the provided data to the underlying NamespaceMerkleTree, and +// automatically uses the first DefaultNamespaceIDLen number of bytes as the +// namespace unless the data pushed to the second half of the tree. Fulfills the +// rsmt.Tree interface. NOTE: panics if an error is encountered while pushing or +// if the tree size is exceeded. +func (w *ErasuredNamespacedMerkleTree) Push(data []byte) error { + if w.axisIndex+1 > 2*w.squareSize || w.shareIndex+1 > 2*w.squareSize { + return fmt.Errorf("pushed past predetermined square size: boundary at %d index at %d %d", 2*w.squareSize, w.axisIndex, w.shareIndex) + } + if len(data) < NamespaceSize { + return fmt.Errorf("data is too short to contain namespace ID") + } + nidAndData := make([]byte, NamespaceSize+len(data)) + copy(nidAndData[NamespaceSize:], data) + // use the parity namespace if the cell is not in Q0 of the extended data square + if w.isQuadrantZero() { + copy(nidAndData[:NamespaceSize], data[:NamespaceSize]) + } else { + copy(nidAndData[:NamespaceSize], ParitySharesNamespaceBytes) + } + err := w.tree.Push(nidAndData) + if err != nil { + return err + } + w.incrementShareIndex() + return nil +} + +// Root fulfills the rsmt.Tree interface by generating and returning the +// underlying NamespaceMerkleTree Root. +func (w *ErasuredNamespacedMerkleTree) Root() ([]byte, error) { + root, err := w.tree.Root() + if err != nil { + return nil, err + } + return root, nil +} + +// incrementShareIndex increments the share index by one. +func (w *ErasuredNamespacedMerkleTree) incrementShareIndex() { + w.shareIndex++ +} + +// isQuadrantZero returns true if the current share index and axis index are both +// in the original data square. +func (w *ErasuredNamespacedMerkleTree) isQuadrantZero() bool { + return w.shareIndex < w.squareSize && w.axisIndex < w.squareSize +} From 5cf9a86569b7ce3b257e0515ec491d1e05b8fe8f Mon Sep 17 00:00:00 2001 From: sanaz Date: Thu, 6 Jul 2023 12:39:00 -0700 Subject: [PATCH 15/39] adds some incomplete tests --- extendeddatacrossword_test.go | 65 +++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index 9524a80..78cb00d 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -343,3 +343,68 @@ func createTestEds(codec Codec, shareSize int) *ExtendedDataSquare { return eds } + +func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { + shareSize := 64 + // corruptChunk := bytes.Repeat([]byte{66}, shareSize) + + tests := []struct { + name string + coords [][]uint + values [][]byte + }{ + { + name: "no corruption", + coords: [][]uint{}, + values: [][]byte{}, + }, + } + + for codecName, codec := range codecs { + t.Run(codecName, func(t *testing.T) { + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + eds := createTestEdsWithNMT(codec, shareSize) + assert.NotNil(t, eds) + rowRoots, err := eds.getRowRoots() + assert.NoError(t, err) + + colRoots, err := eds.getColRoots() + assert.NoError(t, err) + + for i, coords := range test.coords { + x := coords[0] + y := coords[1] + eds.setCell(x, y, test.values[i]) + } + + err = eds.Repair(rowRoots, colRoots) + assert.NoError(t, err) + + }) + } + }) + } +} + +func createTestEdsWithNMT(codec Codec, shareSize int) *ExtendedDataSquare { + ones := bytes.Repeat([]byte{1}, shareSize) + twos := bytes.Repeat([]byte{2}, shareSize) + threes := bytes.Repeat([]byte{3}, shareSize) + fours := bytes.Repeat([]byte{4}, shareSize) + // edsWidth := 4 // number of shares per row/column in the extended data square + // odsWidth := edsWidth / 2 // number of shares per row/column in the original data square + + eds, err := ComputeExtendedDataSquare([][]byte{ + ones, twos, threes, fours, + // original data square would be like + // row1: 1 2 + // row2: 3 4 + }, codec, NewDefaultTree) + + if err != nil { + panic(err) + } + + return eds +} From 9546941f17f2a3355401d7786d95c68dacd9e6ea Mon Sep 17 00:00:00 2001 From: sanaz Date: Thu, 6 Jul 2023 13:20:27 -0700 Subject: [PATCH 16/39] updates the test to consume the wrapper implementation --- extendeddatacrossword_test.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index 78cb00d..af236f8 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "fmt" + "github.com/celestiaorg/nmt" "math/rand" "testing" @@ -364,7 +365,7 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { t.Run(codecName, func(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - eds := createTestEdsWithNMT(codec, shareSize) + eds := createTestEdsWithNMT(t, codec, shareSize, 1) assert.NotNil(t, eds) rowRoots, err := eds.getRowRoots() assert.NoError(t, err) @@ -387,20 +388,23 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { } } -func createTestEdsWithNMT(codec Codec, shareSize int) *ExtendedDataSquare { +func createTestEdsWithNMT(t *testing.T, codec Codec, shareSize, namespaceSize int) *ExtendedDataSquare { + // the first namespaceSize bytes of each share are the namespace + // assert.True(t, shareSize > namespaceSize) + ones := bytes.Repeat([]byte{1}, shareSize) twos := bytes.Repeat([]byte{2}, shareSize) threes := bytes.Repeat([]byte{3}, shareSize) fours := bytes.Repeat([]byte{4}, shareSize) - // edsWidth := 4 // number of shares per row/column in the extended data square - // odsWidth := edsWidth / 2 // number of shares per row/column in the original data square + edsWidth := 4 // number of shares per row/column in the extended data square + odsWidth := edsWidth / 2 // number of shares per row/column in the original data square eds, err := ComputeExtendedDataSquare([][]byte{ ones, twos, threes, fours, // original data square would be like // row1: 1 2 // row2: 3 4 - }, codec, NewDefaultTree) + }, codec, NewConstructor(uint64(odsWidth), nmt.NamespaceIDSize(namespaceSize))) if err != nil { panic(err) From ace3ea5c907c85462b64a1252e7057b3ba64ecb8 Mon Sep 17 00:00:00 2001 From: sanaz Date: Thu, 6 Jul 2023 13:20:42 -0700 Subject: [PATCH 17/39] refactors the wrapper to access to a custom namespace size --- go.mod | 4 +++- go.sum | 4 ++++ nmtwrapper.go | 26 ++++++++++++++++---------- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index a12ffda..4d60a05 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 github.com/celestiaorg/nmt v0.17.0 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.4 ) require ( @@ -13,6 +13,8 @@ require ( golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 ) +require github.com/stretchr/objx v0.5.0 // indirect + require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/klauspost/cpuid/v2 v2.1.1 // indirect diff --git a/go.sum b/go.sum index 6c6e2b8..17a94f1 100644 --- a/go.sum +++ b/go.sum @@ -23,12 +23,15 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= gitlab.com/NebulousLabs/errors v0.0.0-20171229012116-7ead97ef90b8/go.mod h1:ZkMZ0dpQyWwlENaeZVBiQRjhMEZvk6VTXquzl3FOFP8= gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975 h1:L/ENs/Ar1bFzUeKx6m3XjlmBgIUlykX9dzvp5k9NGxc= gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975/go.mod h1:ZkMZ0dpQyWwlENaeZVBiQRjhMEZvk6VTXquzl3FOFP8= @@ -53,4 +56,5 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/nmtwrapper.go b/nmtwrapper.go index 237138f..a642a03 100644 --- a/nmtwrapper.go +++ b/nmtwrapper.go @@ -13,7 +13,7 @@ var ( _ Tree = &ErasuredNamespacedMerkleTree{} ) -const NamespaceSize = 32 +// const NamespaceSize = 32 var ParitySharesNamespaceBytes = []byte{1} @@ -36,10 +36,11 @@ type ErasuredNamespacedMerkleTree struct { // 2*squareSize. shareIndex is used to help determine which quadrant each // leaf belongs to, along with keeping track of how many leaves have been // added to the tree so far. - shareIndex uint64 + shareIndex uint64 + namespaceSize int } -// Tree is an interface that wraps the methods of the underlying +// NMTTree is an interface that wraps the methods of the underlying // NamespaceMerkleTree that are used by ErasuredNamespacedMerkleTree. This // interface is mainly used for testing. It is not recommended to use this // interface by implementing a different implementation. @@ -56,10 +57,15 @@ func NewErasuredNamespacedMerkleTree(squareSize uint64, axisIndex uint, options if squareSize == 0 { panic("cannot create a ErasuredNamespacedMerkleTree of squareSize == 0") } - options = append(options, nmt.NamespaceIDSize(NamespaceSize)) + // options = append(options, nmt.NamespaceIDSize(NamespaceSize)) + // read the options to extract the namespace size + opts := &nmt.Options{} + for _, setter := range options { + setter(opts) + } options = append(options, nmt.IgnoreMaxNamespace(true)) tree := nmt.New(sha256.New(), options...) - return ErasuredNamespacedMerkleTree{squareSize: squareSize, options: options, tree: tree, axisIndex: uint64(axisIndex), shareIndex: 0} + return ErasuredNamespacedMerkleTree{squareSize: squareSize, namespaceSize: int(opts.NamespaceIDSize), options: options, tree: tree, axisIndex: uint64(axisIndex), shareIndex: 0} } type constructor struct { @@ -94,16 +100,16 @@ func (w *ErasuredNamespacedMerkleTree) Push(data []byte) error { if w.axisIndex+1 > 2*w.squareSize || w.shareIndex+1 > 2*w.squareSize { return fmt.Errorf("pushed past predetermined square size: boundary at %d index at %d %d", 2*w.squareSize, w.axisIndex, w.shareIndex) } - if len(data) < NamespaceSize { + if len(data) < w.namespaceSize { return fmt.Errorf("data is too short to contain namespace ID") } - nidAndData := make([]byte, NamespaceSize+len(data)) - copy(nidAndData[NamespaceSize:], data) + nidAndData := make([]byte, w.namespaceSize+len(data)) + copy(nidAndData[w.namespaceSize:], data) // use the parity namespace if the cell is not in Q0 of the extended data square if w.isQuadrantZero() { - copy(nidAndData[:NamespaceSize], data[:NamespaceSize]) + copy(nidAndData[:w.namespaceSize], data[:w.namespaceSize]) } else { - copy(nidAndData[:NamespaceSize], ParitySharesNamespaceBytes) + copy(nidAndData[:w.namespaceSize], ParitySharesNamespaceBytes) } err := w.tree.Push(nidAndData) if err != nil { From c0f00d8bf103049900f9fa121e243542f9bb08da Mon Sep 17 00:00:00 2001 From: sanaz Date: Thu, 6 Jul 2023 13:42:08 -0700 Subject: [PATCH 18/39] sets proper value for parity share namespace --- nmtwrapper.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nmtwrapper.go b/nmtwrapper.go index a642a03..d5b1ce9 100644 --- a/nmtwrapper.go +++ b/nmtwrapper.go @@ -1,6 +1,7 @@ package rsmt2d import ( + "bytes" "fmt" "github.com/celestiaorg/nmt" "github.com/celestiaorg/nmt/namespace" @@ -13,12 +14,10 @@ var ( _ Tree = &ErasuredNamespacedMerkleTree{} ) -// const NamespaceSize = 32 - var ParitySharesNamespaceBytes = []byte{1} // ErasuredNamespacedMerkleTree wraps NamespaceMerkleTree to conform to the -// rsmt2d.Tree interface while also providing the correct namespaces to the +// Tree interface while also providing the correct namespaces to the // underlying NamespaceMerkleTree. It does this by adding the already included // namespace to the first half of the tree, and then uses the parity namespace // ID for each share pushed to the second half of the tree. This allows for the @@ -97,6 +96,7 @@ func (c constructor) NewTree(_ Axis, axisIndex uint) Tree { // rsmt.Tree interface. NOTE: panics if an error is encountered while pushing or // if the tree size is exceeded. func (w *ErasuredNamespacedMerkleTree) Push(data []byte) error { + ParitySharesNamespaceBytes := bytes.Repeat([]byte{0xFF}, w.namespaceSize) if w.axisIndex+1 > 2*w.squareSize || w.shareIndex+1 > 2*w.squareSize { return fmt.Errorf("pushed past predetermined square size: boundary at %d index at %d %d", 2*w.squareSize, w.axisIndex, w.shareIndex) } From 291cae67e52056dca349e26ef9feca2a444b9ef9 Mon Sep 17 00:00:00 2001 From: sanaz Date: Thu, 6 Jul 2023 13:51:53 -0700 Subject: [PATCH 19/39] adds shareValues as argument to the createTestEdsWithNMT --- extendeddatacrossword_test.go | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index af236f8..c221ba4 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -365,7 +365,7 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { t.Run(codecName, func(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - eds := createTestEdsWithNMT(t, codec, shareSize, 1) + eds := createTestEdsWithNMT(t, codec, shareSize, 1, 1, 2, 3, 4) assert.NotNil(t, eds) rowRoots, err := eds.getRowRoots() assert.NoError(t, err) @@ -373,6 +373,7 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { colRoots, err := eds.getColRoots() assert.NoError(t, err) + // corruptEds := createTestEdsWithNMT(t, codec, shareSize, 1) for i, coords := range test.coords { x := coords[0] y := coords[1] @@ -388,23 +389,19 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { } } -func createTestEdsWithNMT(t *testing.T, codec Codec, shareSize, namespaceSize int) *ExtendedDataSquare { +func createTestEdsWithNMT(t *testing.T, codec Codec, shareSize, namespaceSize int, sharesValue ...int) *ExtendedDataSquare { // the first namespaceSize bytes of each share are the namespace // assert.True(t, shareSize > namespaceSize) - ones := bytes.Repeat([]byte{1}, shareSize) - twos := bytes.Repeat([]byte{2}, shareSize) - threes := bytes.Repeat([]byte{3}, shareSize) - fours := bytes.Repeat([]byte{4}, shareSize) + // create shares of shareSize bytes + shares := make([][]byte, len(sharesValue)) + for i, shareValue := range sharesValue { + shares[i] = bytes.Repeat([]byte{byte(shareValue)}, shareSize) + } edsWidth := 4 // number of shares per row/column in the extended data square odsWidth := edsWidth / 2 // number of shares per row/column in the original data square - eds, err := ComputeExtendedDataSquare([][]byte{ - ones, twos, threes, fours, - // original data square would be like - // row1: 1 2 - // row2: 3 4 - }, codec, NewConstructor(uint64(odsWidth), nmt.NamespaceIDSize(namespaceSize))) + eds, err := ComputeExtendedDataSquare(shares, codec, NewConstructor(uint64(odsWidth), nmt.NamespaceIDSize(namespaceSize))) if err != nil { panic(err) From 8ac7b14a650207c31bac9bb80862b0eb5f4dffb7 Mon Sep 17 00:00:00 2001 From: sanaz Date: Thu, 6 Jul 2023 14:07:54 -0700 Subject: [PATCH 20/39] adds tests for unordered shares --- extendeddatacrossword_test.go | 54 +++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index c221ba4..55b2665 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -347,17 +347,32 @@ func createTestEds(codec Codec, shareSize int) *ExtendedDataSquare { func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { shareSize := 64 - // corruptChunk := bytes.Repeat([]byte{66}, shareSize) - + namespaceSize := 1 tests := []struct { - name string - coords [][]uint - values [][]byte + name string + sharesValue []int + wantErr bool }{ { - name: "no corruption", - coords: [][]uint{}, - values: [][]byte{}, + name: "no corruption", + sharesValue: []int{1, 2, 3, 4}, + wantErr: false, + }, + { + // make rows shares unordered, columns are still ordered + // 2 1 + // 4 3 + name: "rows with unordered shares", + sharesValue: []int{2, 1, 4, 3}, + wantErr: true, // repair should error out during root construction + }, + { + // make column shares unordered, rows are still ordered + // 3 4 + // 1 2 + name: "columns with unordered shares", + sharesValue: []int{3, 4, 1, 2}, + wantErr: true, // repair should error out during root construction }, } @@ -365,23 +380,20 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { t.Run(codecName, func(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - eds := createTestEdsWithNMT(t, codec, shareSize, 1, 1, 2, 3, 4) + + // create a DA header + eds := createTestEdsWithNMT(t, codec, shareSize, namespaceSize, 1, 2, 3, 4) assert.NotNil(t, eds) - rowRoots, err := eds.getRowRoots() + dAHeaderRoots, err := eds.getRowRoots() assert.NoError(t, err) - colRoots, err := eds.getColRoots() + dAHeaderCols, err := eds.getColRoots() assert.NoError(t, err) - // corruptEds := createTestEdsWithNMT(t, codec, shareSize, 1) - for i, coords := range test.coords { - x := coords[0] - y := coords[1] - eds.setCell(x, y, test.values[i]) - } - - err = eds.Repair(rowRoots, colRoots) - assert.NoError(t, err) + // create an eds with supposedly sampled shares, which might be corrupted + corruptEds := createTestEdsWithNMT(t, codec, shareSize, namespaceSize, test.sharesValue...) + err = corruptEds.Repair(dAHeaderRoots, dAHeaderCols) + assert.Equal(t, err != nil, test.wantErr) }) } @@ -391,7 +403,7 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { func createTestEdsWithNMT(t *testing.T, codec Codec, shareSize, namespaceSize int, sharesValue ...int) *ExtendedDataSquare { // the first namespaceSize bytes of each share are the namespace - // assert.True(t, shareSize > namespaceSize) + assert.True(t, shareSize > namespaceSize) // create shares of shareSize bytes shares := make([][]byte, len(sharesValue)) From 7d464ebcb2ca2132d1a4a687c030fa00cd7020c0 Mon Sep 17 00:00:00 2001 From: sanaz Date: Thu, 6 Jul 2023 16:16:31 -0700 Subject: [PATCH 21/39] adds further criteria to the tests --- extendeddatacrossword_test.go | 76 ++++++++++++----------------------- 1 file changed, 25 insertions(+), 51 deletions(-) diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index 31817c5..9a66604 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" "github.com/celestiaorg/nmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "math/rand" "testing" ) @@ -258,48 +260,6 @@ func TestCorruptedEdsReturnsErrByzantineData(t *testing.T) { } } -func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { - shareSize := 64 - // corruptChunk := bytes.Repeat([]byte{66}, shareSize) - - tests := []struct { - name string - coords [][]uint - values [][]byte - }{ - { - name: "no corruption", - coords: [][]uint{}, - values: [][]byte{}, - }, - } - - for codecName, codec := range codecs { - t.Run(codecName, func(t *testing.T) { - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - eds := createTestEdsWithNMT(codec, shareSize) - rowRoots, err := eds.getRowRoots() - assert.NoError(t, err) - - colRoots, err := eds.getColRoots() - assert.NoError(t, err) - - for i, coords := range test.coords { - x := coords[0] - y := coords[1] - eds.setCell(x, y, test.values[i]) - } - - err = eds.Repair(rowRoots, colRoots) - assert.NoError(t, err) - - }) - } - }) - } -} - func BenchmarkRepair(b *testing.B) { // For different ODS sizes for originalDataWidth := 4; originalDataWidth <= 512; originalDataWidth *= 2 { @@ -388,30 +348,37 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { shareSize := 64 namespaceSize := 1 tests := []struct { - name string - sharesValue []int - wantErr bool + name string + sharesValue []int + wantErr bool + errType error + corruptedAxis Axis }{ { name: "no corruption", sharesValue: []int{1, 2, 3, 4}, wantErr: false, + errType: nil, }, { // make rows shares unordered, columns are still ordered // 2 1 // 4 3 - name: "rows with unordered shares", - sharesValue: []int{2, 1, 4, 3}, - wantErr: true, // repair should error out during root construction + name: "rows with unordered shares", + sharesValue: []int{2, 1, 4, 3}, + wantErr: true, // repair should error out during root construction + errType: &ErrByzantineData{}, + corruptedAxis: Row, }, { // make column shares unordered, rows are still ordered // 3 4 // 1 2 - name: "columns with unordered shares", - sharesValue: []int{3, 4, 1, 2}, - wantErr: true, // repair should error out during root construction + name: "columns with unordered shares", + sharesValue: []int{3, 4, 1, 2}, + wantErr: true, // repair should error out during root construction + errType: &ErrByzantineData{}, + corruptedAxis: Col, }, } @@ -431,8 +398,15 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { // create an eds with supposedly sampled shares, which might be corrupted corruptEds := createTestEdsWithNMT(t, codec, shareSize, namespaceSize, test.sharesValue...) + assert.NotNil(t, eds) err = corruptEds.Repair(dAHeaderRoots, dAHeaderCols) assert.Equal(t, err != nil, test.wantErr) + if test.wantErr { + assert.ErrorAs(t, err, &test.errType) + var byzErr *ErrByzantineData + errors.As(err, &byzErr) + // assert.Equal(t, byzErr.Axis, test.corruptedAxis) + } }) } From d3101f594d5fc360cd73c8da492b1d117a7f9d1b Mon Sep 17 00:00:00 2001 From: sanaz Date: Fri, 7 Jul 2023 07:55:45 -0700 Subject: [PATCH 22/39] includes test description --- extendeddatacrossword_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index 9a66604..359b6b2 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -414,6 +414,10 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { } } +// createTestEdsWithNMT creates an extended data square with the given shares and namespace size. +// Shares are placed in row-major order. +// The first namespaceSize bytes of each share are treated as its namespace. +// Roots of the extended data square are computed using namespace merkle trees. func createTestEdsWithNMT(t *testing.T, codec Codec, shareSize, namespaceSize int, sharesValue ...int) *ExtendedDataSquare { // the first namespaceSize bytes of each share are the namespace assert.True(t, shareSize > namespaceSize) @@ -426,7 +430,7 @@ func createTestEdsWithNMT(t *testing.T, codec Codec, shareSize, namespaceSize in edsWidth := 4 // number of shares per row/column in the extended data square odsWidth := edsWidth / 2 // number of shares per row/column in the original data square - eds, err := ComputeExtendedDataSquare(shares, codec, NewConstructor(uint64(odsWidth), nmt.NamespaceIDSize(namespaceSize))) + eds, err := ComputeExtendedDataSquare(shares, codec, newConstructor(uint64(odsWidth), nmt.NamespaceIDSize(namespaceSize))) if err != nil { panic(err) From d4a564b41ed30d41673680bebb693901cb294b62 Mon Sep 17 00:00:00 2001 From: sanaz Date: Fri, 7 Jul 2023 07:56:07 -0700 Subject: [PATCH 23/39] makes nmtwrapper types and functions un-importable --- nmtwrapper.go | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/nmtwrapper.go b/nmtwrapper.go index d5b1ce9..a3f00ed 100644 --- a/nmtwrapper.go +++ b/nmtwrapper.go @@ -10,23 +10,21 @@ import ( // Fulfills the Tree interface and TreeConstructorFn function var ( - _ TreeConstructorFn = NewConstructor(0) - _ Tree = &ErasuredNamespacedMerkleTree{} + _ TreeConstructorFn = newConstructor(0) + _ Tree = &erasuredNamespacedMerkleTree{} ) -var ParitySharesNamespaceBytes = []byte{1} - -// ErasuredNamespacedMerkleTree wraps NamespaceMerkleTree to conform to the +// erasuredNamespacedMerkleTree wraps NamespaceMerkleTree to conform to the // Tree interface while also providing the correct namespaces to the // underlying NamespaceMerkleTree. It does this by adding the already included // namespace to the first half of the tree, and then uses the parity namespace // ID for each share pushed to the second half of the tree. This allows for the // namespaces to be included in the erasure data, while also keeping the nmt // library sufficiently general -type ErasuredNamespacedMerkleTree struct { +type erasuredNamespacedMerkleTree struct { squareSize uint64 // note: this refers to the width of the original square before erasure-coded options []nmt.Option - tree NMTTree + tree nmtTree // axisIndex is the index of the axis (row or column) that this tree is on. This is passed // by rsmt2d and used to help determine which quadrant each leaf belongs to. axisIndex uint64 @@ -39,22 +37,22 @@ type ErasuredNamespacedMerkleTree struct { namespaceSize int } -// NMTTree is an interface that wraps the methods of the underlying -// NamespaceMerkleTree that are used by ErasuredNamespacedMerkleTree. This +// nmtTree is an interface that wraps the methods of the underlying +// NamespaceMerkleTree that are used by erasuredNamespacedMerkleTree. This // interface is mainly used for testing. It is not recommended to use this // interface by implementing a different implementation. -type NMTTree interface { +type nmtTree interface { Root() ([]byte, error) Push(namespacedData namespace.PrefixedData) error } -// NewErasuredNamespacedMerkleTree creates a new ErasuredNamespacedMerkleTree +// newErasuredNamespacedMerkleTree creates a new erasuredNamespacedMerkleTree // with an underlying NMT of namespace size `NamespaceSize` and with // `ignoreMaxNamespace=true`. axisIndex is the index of the row or column that // this tree is committing to. squareSize must be greater than zero. -func NewErasuredNamespacedMerkleTree(squareSize uint64, axisIndex uint, options ...nmt.Option) ErasuredNamespacedMerkleTree { +func newErasuredNamespacedMerkleTree(squareSize uint64, axisIndex uint, options ...nmt.Option) erasuredNamespacedMerkleTree { if squareSize == 0 { - panic("cannot create a ErasuredNamespacedMerkleTree of squareSize == 0") + panic("cannot create a erasuredNamespacedMerkleTree of squareSize == 0") } // options = append(options, nmt.NamespaceIDSize(NamespaceSize)) // read the options to extract the namespace size @@ -64,7 +62,7 @@ func NewErasuredNamespacedMerkleTree(squareSize uint64, axisIndex uint, options } options = append(options, nmt.IgnoreMaxNamespace(true)) tree := nmt.New(sha256.New(), options...) - return ErasuredNamespacedMerkleTree{squareSize: squareSize, namespaceSize: int(opts.NamespaceIDSize), options: options, tree: tree, axisIndex: uint64(axisIndex), shareIndex: 0} + return erasuredNamespacedMerkleTree{squareSize: squareSize, namespaceSize: int(opts.NamespaceIDSize), options: options, tree: tree, axisIndex: uint64(axisIndex), shareIndex: 0} } type constructor struct { @@ -72,10 +70,10 @@ type constructor struct { opts []nmt.Option } -// NewConstructor creates a tree constructor function as required by rsmt2d to +// newConstructor creates a tree constructor function as required by rsmt2d to // calculate the data root. It creates that tree using the -// wrapper.ErasuredNamespacedMerkleTree. -func NewConstructor(squareSize uint64, opts ...nmt.Option) TreeConstructorFn { +// wrapper.erasuredNamespacedMerkleTree. +func newConstructor(squareSize uint64, opts ...nmt.Option) TreeConstructorFn { return constructor{ squareSize: squareSize, opts: opts, @@ -83,10 +81,10 @@ func NewConstructor(squareSize uint64, opts ...nmt.Option) TreeConstructorFn { } // NewTree creates a new Tree using the -// ErasuredNamespacedMerkleTree with predefined square size and +// erasuredNamespacedMerkleTree with predefined square size and // nmt.Options func (c constructor) NewTree(_ Axis, axisIndex uint) Tree { - newTree := NewErasuredNamespacedMerkleTree(c.squareSize, axisIndex, c.opts...) + newTree := newErasuredNamespacedMerkleTree(c.squareSize, axisIndex, c.opts...) return &newTree } @@ -95,7 +93,7 @@ func (c constructor) NewTree(_ Axis, axisIndex uint) Tree { // namespace unless the data pushed to the second half of the tree. Fulfills the // rsmt.Tree interface. NOTE: panics if an error is encountered while pushing or // if the tree size is exceeded. -func (w *ErasuredNamespacedMerkleTree) Push(data []byte) error { +func (w *erasuredNamespacedMerkleTree) Push(data []byte) error { ParitySharesNamespaceBytes := bytes.Repeat([]byte{0xFF}, w.namespaceSize) if w.axisIndex+1 > 2*w.squareSize || w.shareIndex+1 > 2*w.squareSize { return fmt.Errorf("pushed past predetermined square size: boundary at %d index at %d %d", 2*w.squareSize, w.axisIndex, w.shareIndex) @@ -121,7 +119,7 @@ func (w *ErasuredNamespacedMerkleTree) Push(data []byte) error { // Root fulfills the rsmt.Tree interface by generating and returning the // underlying NamespaceMerkleTree Root. -func (w *ErasuredNamespacedMerkleTree) Root() ([]byte, error) { +func (w *erasuredNamespacedMerkleTree) Root() ([]byte, error) { root, err := w.tree.Root() if err != nil { return nil, err @@ -130,12 +128,12 @@ func (w *ErasuredNamespacedMerkleTree) Root() ([]byte, error) { } // incrementShareIndex increments the share index by one. -func (w *ErasuredNamespacedMerkleTree) incrementShareIndex() { +func (w *erasuredNamespacedMerkleTree) incrementShareIndex() { w.shareIndex++ } // isQuadrantZero returns true if the current share index and axis index are both // in the original data square. -func (w *ErasuredNamespacedMerkleTree) isQuadrantZero() bool { +func (w *erasuredNamespacedMerkleTree) isQuadrantZero() bool { return w.shareIndex < w.squareSize && w.axisIndex < w.squareSize } From 145ec9f88ee4bc5e021ef8d7e17be7e660bff7f2 Mon Sep 17 00:00:00 2001 From: sanaz Date: Fri, 7 Jul 2023 08:08:44 -0700 Subject: [PATCH 24/39] adds godoc for newDataSquare --- datasquare.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datasquare.go b/datasquare.go index 4a5ca0c..e12cfbc 100644 --- a/datasquare.go +++ b/datasquare.go @@ -26,6 +26,9 @@ type dataSquare struct { createTreeFn TreeConstructorFn } +// newDataSquare populates the data square from the supplied data and treeCreator. +// No root calculation is performed. +// data may have nil values. func newDataSquare(data [][]byte, treeCreator TreeConstructorFn) (*dataSquare, error) { width := int(math.Ceil(math.Sqrt(float64(len(data))))) if width*width != len(data) { From 90e2060f5a2190f3e84d19339ead76d854aa3759 Mon Sep 17 00:00:00 2001 From: sanaz Date: Fri, 7 Jul 2023 08:09:05 -0700 Subject: [PATCH 25/39] revises godocs of nmtwrapper file --- nmtwrapper.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/nmtwrapper.go b/nmtwrapper.go index a3f00ed..2208e50 100644 --- a/nmtwrapper.go +++ b/nmtwrapper.go @@ -54,8 +54,7 @@ func newErasuredNamespacedMerkleTree(squareSize uint64, axisIndex uint, options if squareSize == 0 { panic("cannot create a erasuredNamespacedMerkleTree of squareSize == 0") } - // options = append(options, nmt.NamespaceIDSize(NamespaceSize)) - // read the options to extract the namespace size + // read the options to extract the namespace size, and use it to construct erasuredNamespacedMerkleTree opts := &nmt.Options{} for _, setter := range options { setter(opts) @@ -72,7 +71,7 @@ type constructor struct { // newConstructor creates a tree constructor function as required by rsmt2d to // calculate the data root. It creates that tree using the -// wrapper.erasuredNamespacedMerkleTree. +// erasuredNamespacedMerkleTree. func newConstructor(squareSize uint64, opts ...nmt.Option) TreeConstructorFn { return constructor{ squareSize: squareSize, @@ -89,9 +88,9 @@ func (c constructor) NewTree(_ Axis, axisIndex uint) Tree { } // Push adds the provided data to the underlying NamespaceMerkleTree, and -// automatically uses the first DefaultNamespaceIDLen number of bytes as the +// automatically uses the first erasuredNamespacedMerkleTree.namespaceSize number of bytes as the // namespace unless the data pushed to the second half of the tree. Fulfills the -// rsmt.Tree interface. NOTE: panics if an error is encountered while pushing or +// rsmt2d.Tree interface. NOTE: panics if an error is encountered while pushing or // if the tree size is exceeded. func (w *erasuredNamespacedMerkleTree) Push(data []byte) error { ParitySharesNamespaceBytes := bytes.Repeat([]byte{0xFF}, w.namespaceSize) @@ -117,7 +116,7 @@ func (w *erasuredNamespacedMerkleTree) Push(data []byte) error { return nil } -// Root fulfills the rsmt.Tree interface by generating and returning the +// Root fulfills the rsmt2d.Tree interface by generating and returning the // underlying NamespaceMerkleTree Root. func (w *erasuredNamespacedMerkleTree) Root() ([]byte, error) { root, err := w.tree.Root() From 0df133a17d8c0437cfe613c1f59c911f3f3a1e3d Mon Sep 17 00:00:00 2001 From: sanaz Date: Fri, 7 Jul 2023 10:24:09 -0700 Subject: [PATCH 26/39] creates DA header once --- extendeddatacrossword_test.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index 359b6b2..452e205 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -384,18 +384,17 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { for codecName, codec := range codecs { t.Run(codecName, func(t *testing.T) { + // create a DA header + eds := createTestEdsWithNMT(t, codec, shareSize, namespaceSize, 1, 2, 3, 4) + assert.NotNil(t, eds) + dAHeaderRoots, err := eds.getRowRoots() + assert.NoError(t, err) + + dAHeaderCols, err := eds.getColRoots() + assert.NoError(t, err) for _, test := range tests { t.Run(test.name, func(t *testing.T) { - // create a DA header - eds := createTestEdsWithNMT(t, codec, shareSize, namespaceSize, 1, 2, 3, 4) - assert.NotNil(t, eds) - dAHeaderRoots, err := eds.getRowRoots() - assert.NoError(t, err) - - dAHeaderCols, err := eds.getColRoots() - assert.NoError(t, err) - // create an eds with supposedly sampled shares, which might be corrupted corruptEds := createTestEdsWithNMT(t, codec, shareSize, namespaceSize, test.sharesValue...) assert.NotNil(t, eds) From 984845833959001925fc7a92c201bcfc58f3835d Mon Sep 17 00:00:00 2001 From: sanaz Date: Fri, 7 Jul 2023 11:10:13 -0700 Subject: [PATCH 27/39] adds corruption coordinates and corrupted index --- extendeddatacrossword_test.go | 45 +++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index 452e205..9285eb1 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -347,12 +347,19 @@ func createTestEds(codec Codec, shareSize int) *ExtendedDataSquare { func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { shareSize := 64 namespaceSize := 1 + one := bytes.Repeat([]byte{1}, shareSize) + two := bytes.Repeat([]byte{2}, shareSize) + three := bytes.Repeat([]byte{3}, shareSize) + // four := bytes.Repeat([]byte{4}, shareSize) tests := []struct { name string sharesValue []int + coords [][]uint + values [][]byte wantErr bool errType error corruptedAxis Axis + corruptedInd int }{ { name: "no corruption", @@ -363,22 +370,40 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { { // make rows shares unordered, columns are still ordered // 2 1 - // 4 3 + // - - name: "rows with unordered shares", - sharesValue: []int{2, 1, 4, 3}, + sharesValue: []int{1, 2, 3, 4}, wantErr: true, // repair should error out during root construction errType: &ErrByzantineData{}, corruptedAxis: Row, + coords: [][]uint{{0, 0}, {0, 1}, + {1, 0}, {1, 1}, {1, 2}, {1, 3}, + {2, 0}, {2, 1}, {2, 2}, {2, 3}, + {3, 0}, {3, 1}, {3, 2}, {3, 3}}, + values: [][]byte{two, one, + nil, nil, nil, nil, + nil, nil, nil, nil, + nil, nil, nil, nil}, + corruptedInd: 0, }, { // make column shares unordered, rows are still ordered - // 3 4 - // 1 2 + // 3 - + // 1 - name: "columns with unordered shares", - sharesValue: []int{3, 4, 1, 2}, + sharesValue: []int{1, 2, 3, 4}, wantErr: true, // repair should error out during root construction errType: &ErrByzantineData{}, corruptedAxis: Col, + coords: [][]uint{{0, 0}, {0, 1}, {0, 2}, {0, 3}, + {1, 0}, {1, 1}, {1, 2}, {1, 3}, + {2, 1}, {2, 2}, {2, 3}, + {3, 1}, {3, 2}, {3, 3}}, + values: [][]byte{three, nil, nil, nil, + one, nil, nil, nil, + nil, nil, nil, + nil, nil, nil}, + corruptedInd: 0, }, } @@ -394,17 +419,23 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { assert.NoError(t, err) for _, test := range tests { t.Run(test.name, func(t *testing.T) { - // create an eds with supposedly sampled shares, which might be corrupted corruptEds := createTestEdsWithNMT(t, codec, shareSize, namespaceSize, test.sharesValue...) assert.NotNil(t, eds) + + for i, coords := range test.coords { + x := coords[0] + y := coords[1] + corruptEds.setCell(x, y, test.values[i]) + } + err = corruptEds.Repair(dAHeaderRoots, dAHeaderCols) assert.Equal(t, err != nil, test.wantErr) if test.wantErr { assert.ErrorAs(t, err, &test.errType) var byzErr *ErrByzantineData errors.As(err, &byzErr) - // assert.Equal(t, byzErr.Axis, test.corruptedAxis) + assert.Equal(t, byzErr.Axis, test.corruptedAxis) } }) From 5619a748bd52413ec4a05494f6e17abac013f6a0 Mon Sep 17 00:00:00 2001 From: sanaz Date: Fri, 7 Jul 2023 12:27:44 -0700 Subject: [PATCH 28/39] resolves linter issues --- extendeddatacrossword.go | 1 + extendeddatacrossword_test.go | 59 +++++++++++++++++++++++++---------- nmtwrapper.go | 1 + 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/extendeddatacrossword.go b/extendeddatacrossword.go index e8b8ab6..21bf808 100644 --- a/extendeddatacrossword.go +++ b/extendeddatacrossword.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "golang.org/x/sync/errgroup" ) diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index 5902367..6f1b17e 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -4,11 +4,12 @@ import ( "bytes" "errors" "fmt" + "math/rand" + "testing" + "github.com/celestiaorg/nmt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "math/rand" - "testing" ) // PseudoFraudProof is an example fraud proof. @@ -377,14 +378,28 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { wantErr: true, // repair should error out during root construction errType: &ErrByzantineData{}, corruptedAxis: Row, - coords: [][]uint{{0, 0}, {0, 1}, - {1, 0}, {1, 1}, {1, 2}, {1, 3}, - {2, 0}, {2, 1}, {2, 2}, {2, 3}, - {3, 0}, {3, 1}, {3, 2}, {3, 3}}, - values: [][]byte{two, one, + coords: [][]uint{ + {0, 0}, + {0, 1}, + {1, 0}, + {1, 1}, + {1, 2}, + {1, 3}, + {2, 0}, + {2, 1}, + {2, 2}, + {2, 3}, + {3, 0}, + {3, 1}, + {3, 2}, + {3, 3}, + }, + values: [][]byte{ + two, one, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil}, + nil, nil, nil, nil, + }, corruptedInd: 0, }, { @@ -396,14 +411,28 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { wantErr: true, // repair should error out during root construction errType: &ErrByzantineData{}, corruptedAxis: Col, - coords: [][]uint{{0, 0}, {0, 1}, {0, 2}, {0, 3}, - {1, 0}, {1, 1}, {1, 2}, {1, 3}, - {2, 1}, {2, 2}, {2, 3}, - {3, 1}, {3, 2}, {3, 3}}, - values: [][]byte{three, nil, nil, nil, + coords: [][]uint{ + {0, 0}, + {0, 1}, + {0, 2}, + {0, 3}, + {1, 0}, + {1, 1}, + {1, 2}, + {1, 3}, + {2, 1}, + {2, 2}, + {2, 3}, + {3, 1}, + {3, 2}, + {3, 3}, + }, + values: [][]byte{ + three, nil, nil, nil, one, nil, nil, nil, nil, nil, nil, - nil, nil, nil}, + nil, nil, nil, + }, corruptedInd: 0, }, } @@ -438,7 +467,6 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { errors.As(err, &byzErr) assert.Equal(t, byzErr.Axis, test.corruptedAxis) } - }) } }) @@ -462,7 +490,6 @@ func createTestEdsWithNMT(t *testing.T, codec Codec, shareSize, namespaceSize in odsWidth := edsWidth / 2 // number of shares per row/column in the original data square eds, err := ComputeExtendedDataSquare(shares, codec, newConstructor(uint64(odsWidth), nmt.NamespaceIDSize(namespaceSize))) - if err != nil { panic(err) } diff --git a/nmtwrapper.go b/nmtwrapper.go index 2208e50..1a29635 100644 --- a/nmtwrapper.go +++ b/nmtwrapper.go @@ -3,6 +3,7 @@ package rsmt2d import ( "bytes" "fmt" + "github.com/celestiaorg/nmt" "github.com/celestiaorg/nmt/namespace" "github.com/minio/sha256-simd" From 97af7e2e5b782c97cc9bd1c9241c9706acb4c028 Mon Sep 17 00:00:00 2001 From: sanaz Date: Fri, 7 Jul 2023 12:39:52 -0700 Subject: [PATCH 29/39] reformats the code and adds check for the corrupted index --- extendeddatacrossword.go | 2 +- extendeddatacrossword_test.go | 54 +++++++++-------------------------- 2 files changed, 15 insertions(+), 41 deletions(-) diff --git a/extendeddatacrossword.go b/extendeddatacrossword.go index 21bf808..47b47f3 100644 --- a/extendeddatacrossword.go +++ b/extendeddatacrossword.go @@ -434,7 +434,7 @@ func noMissingData(input [][]byte, rebuiltIndex int) bool { return true } -// computeSharesRoot computes the root of the shares of the specified axis (column or row) and index `i`. +// computeSharesRoot calculates the root of the shares for the specified axis (`i`th column or row). func (eds *ExtendedDataSquare) computeSharesRoot(shares [][]byte, axis Axis, i uint) ([]byte, error) { tree := eds.createTreeFn(axis, i) for _, d := range shares { diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index 6f1b17e..0d38254 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -361,38 +361,24 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { wantErr bool errType error corruptedAxis Axis - corruptedInd int + corruptedInd uint }{ { name: "no corruption", sharesValue: []int{1, 2, 3, 4}, wantErr: false, - errType: nil, }, { - // make rows shares unordered, columns are still ordered - // 2 1 - // - - + // disturbs the order of shares in the first row, erases the rest of the eds name: "rows with unordered shares", sharesValue: []int{1, 2, 3, 4}, wantErr: true, // repair should error out during root construction errType: &ErrByzantineData{}, corruptedAxis: Row, - coords: [][]uint{ - {0, 0}, - {0, 1}, - {1, 0}, - {1, 1}, - {1, 2}, - {1, 3}, - {2, 0}, - {2, 1}, - {2, 2}, - {2, 3}, - {3, 0}, - {3, 1}, - {3, 2}, - {3, 3}, + coords: [][]uint{{0, 0}, {0, 1}, + {1, 0}, {1, 1}, {1, 2}, {1, 3}, + {2, 0}, {2, 1}, {2, 2}, {2, 3}, + {3, 0}, {3, 1}, {3, 2}, {3, 3}, }, values: [][]byte{ two, one, @@ -403,29 +389,16 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { corruptedInd: 0, }, { - // make column shares unordered, rows are still ordered - // 3 - - // 1 - + // disturbs the order of shares in the first column, erases the rest of the eds name: "columns with unordered shares", sharesValue: []int{1, 2, 3, 4}, wantErr: true, // repair should error out during root construction errType: &ErrByzantineData{}, corruptedAxis: Col, - coords: [][]uint{ - {0, 0}, - {0, 1}, - {0, 2}, - {0, 3}, - {1, 0}, - {1, 1}, - {1, 2}, - {1, 3}, - {2, 1}, - {2, 2}, - {2, 3}, - {3, 1}, - {3, 2}, - {3, 3}, + coords: [][]uint{{0, 0}, {0, 1}, {0, 2}, {0, 3}, + {1, 0}, {1, 1}, {1, 2}, {1, 3}, + {2, 1}, {2, 2}, {2, 3}, + {3, 1}, {3, 2}, {3, 3}, }, values: [][]byte{ three, nil, nil, nil, @@ -449,10 +422,10 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { assert.NoError(t, err) for _, test := range tests { t.Run(test.name, func(t *testing.T) { - // create an eds with supposedly sampled shares, which might be corrupted + // create an eds with the given shares corruptEds := createTestEdsWithNMT(t, codec, shareSize, namespaceSize, test.sharesValue...) assert.NotNil(t, eds) - + // corrupt it by setting the values at the given coordinates for i, coords := range test.coords { x := coords[0] y := coords[1] @@ -466,6 +439,7 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { var byzErr *ErrByzantineData errors.As(err, &byzErr) assert.Equal(t, byzErr.Axis, test.corruptedAxis) + assert.Equal(t, byzErr.Index, test.corruptedInd) } }) } From 2146d0e6bc4d8ff75925080e19b8cdb6b2a71661 Mon Sep 17 00:00:00 2001 From: sanaz Date: Fri, 7 Jul 2023 12:47:16 -0700 Subject: [PATCH 30/39] adds some context about the nmtwrapper file --- nmtwrapper.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nmtwrapper.go b/nmtwrapper.go index 1a29635..b9b5b92 100644 --- a/nmtwrapper.go +++ b/nmtwrapper.go @@ -1,5 +1,9 @@ package rsmt2d +// The contents of this file have been adapted from the source file available at https://github.com/celestiaorg/celestia-app/blob/main/pkg/wrapper/nmt_wrapper.go, +// solely for the purpose of testing rsmt2d expected behavior when integrated with a NamespaceMerkleTree. +// Please note that this file has undergone several modifications and may not match the original file exactly. + import ( "bytes" "fmt" @@ -11,8 +15,7 @@ import ( // Fulfills the Tree interface and TreeConstructorFn function var ( - _ TreeConstructorFn = newConstructor(0) - _ Tree = &erasuredNamespacedMerkleTree{} + _ Tree = &erasuredNamespacedMerkleTree{} ) // erasuredNamespacedMerkleTree wraps NamespaceMerkleTree to conform to the From c2a2d4b1e2b2579ec882a08e8b1135ba50ac6f3b Mon Sep 17 00:00:00 2001 From: sanaz Date: Fri, 7 Jul 2023 12:51:29 -0700 Subject: [PATCH 31/39] fixes linter issues --- extendeddatacrossword_test.go | 38 +++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index 0d38254..3c64743 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -375,10 +375,21 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { wantErr: true, // repair should error out during root construction errType: &ErrByzantineData{}, corruptedAxis: Row, - coords: [][]uint{{0, 0}, {0, 1}, - {1, 0}, {1, 1}, {1, 2}, {1, 3}, - {2, 0}, {2, 1}, {2, 2}, {2, 3}, - {3, 0}, {3, 1}, {3, 2}, {3, 3}, + coords: [][]uint{ + {0, 0}, + {0, 1}, + {1, 0}, + {1, 1}, + {1, 2}, + {1, 3}, + {2, 0}, + {2, 1}, + {2, 2}, + {2, 3}, + {3, 0}, + {3, 1}, + {3, 2}, + {3, 3}, }, values: [][]byte{ two, one, @@ -395,10 +406,21 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { wantErr: true, // repair should error out during root construction errType: &ErrByzantineData{}, corruptedAxis: Col, - coords: [][]uint{{0, 0}, {0, 1}, {0, 2}, {0, 3}, - {1, 0}, {1, 1}, {1, 2}, {1, 3}, - {2, 1}, {2, 2}, {2, 3}, - {3, 1}, {3, 2}, {3, 3}, + coords: [][]uint{ + {0, 0}, + {0, 1}, + {0, 2}, + {0, 3}, + {1, 0}, + {1, 1}, + {1, 2}, + {1, 3}, + {2, 1}, + {2, 2}, + {2, 3}, + {3, 1}, + {3, 2}, + {3, 3}, }, values: [][]byte{ three, nil, nil, nil, From c4b41e9e89fc11a30fe97303a1066e3a1d8cb9de Mon Sep 17 00:00:00 2001 From: sanaz Date: Fri, 7 Jul 2023 12:55:29 -0700 Subject: [PATCH 32/39] removes sharesValue from test struct --- extendeddatacrossword_test.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index 3c64743..cadff37 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -352,10 +352,9 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { one := bytes.Repeat([]byte{1}, shareSize) two := bytes.Repeat([]byte{2}, shareSize) three := bytes.Repeat([]byte{3}, shareSize) - // four := bytes.Repeat([]byte{4}, shareSize) + sharesValue := []int{1, 2, 3, 4} tests := []struct { name string - sharesValue []int coords [][]uint values [][]byte wantErr bool @@ -364,14 +363,12 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { corruptedInd uint }{ { - name: "no corruption", - sharesValue: []int{1, 2, 3, 4}, - wantErr: false, + name: "no corruption", + wantErr: false, }, { // disturbs the order of shares in the first row, erases the rest of the eds name: "rows with unordered shares", - sharesValue: []int{1, 2, 3, 4}, wantErr: true, // repair should error out during root construction errType: &ErrByzantineData{}, corruptedAxis: Row, @@ -402,7 +399,6 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { { // disturbs the order of shares in the first column, erases the rest of the eds name: "columns with unordered shares", - sharesValue: []int{1, 2, 3, 4}, wantErr: true, // repair should error out during root construction errType: &ErrByzantineData{}, corruptedAxis: Col, @@ -445,7 +441,7 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { // create an eds with the given shares - corruptEds := createTestEdsWithNMT(t, codec, shareSize, namespaceSize, test.sharesValue...) + corruptEds := createTestEdsWithNMT(t, codec, shareSize, namespaceSize, sharesValue...) assert.NotNil(t, eds) // corrupt it by setting the values at the given coordinates for i, coords := range test.coords { From 976934ae4569a533b00231c56e1d705e706439e3 Mon Sep 17 00:00:00 2001 From: sanaz Date: Fri, 7 Jul 2023 13:04:21 -0700 Subject: [PATCH 33/39] removes error type as all tests have identical error type --- extendeddatacrossword_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index cadff37..bcff2f4 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -358,7 +358,6 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { coords [][]uint values [][]byte wantErr bool - errType error corruptedAxis Axis corruptedInd uint }{ @@ -370,7 +369,6 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { // disturbs the order of shares in the first row, erases the rest of the eds name: "rows with unordered shares", wantErr: true, // repair should error out during root construction - errType: &ErrByzantineData{}, corruptedAxis: Row, coords: [][]uint{ {0, 0}, @@ -400,7 +398,6 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { // disturbs the order of shares in the first column, erases the rest of the eds name: "columns with unordered shares", wantErr: true, // repair should error out during root construction - errType: &ErrByzantineData{}, corruptedAxis: Col, coords: [][]uint{ {0, 0}, @@ -453,8 +450,8 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { err = corruptEds.Repair(dAHeaderRoots, dAHeaderCols) assert.Equal(t, err != nil, test.wantErr) if test.wantErr { - assert.ErrorAs(t, err, &test.errType) var byzErr *ErrByzantineData + assert.ErrorAs(t, err, byzErr) errors.As(err, &byzErr) assert.Equal(t, byzErr.Axis, test.corruptedAxis) assert.Equal(t, byzErr.Index, test.corruptedInd) From a525e499467ba5a44cd89f5a26f8f02e13988008 Mon Sep 17 00:00:00 2001 From: sanaz Date: Fri, 7 Jul 2023 13:09:01 -0700 Subject: [PATCH 34/39] fixes a bug --- extendeddatacrossword_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index bcff2f4..a63a1ac 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -451,7 +451,7 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { assert.Equal(t, err != nil, test.wantErr) if test.wantErr { var byzErr *ErrByzantineData - assert.ErrorAs(t, err, byzErr) + assert.ErrorAs(t, err, &byzErr) errors.As(err, &byzErr) assert.Equal(t, byzErr.Axis, test.corruptedAxis) assert.Equal(t, byzErr.Index, test.corruptedInd) From 2dcfb41c7bd30da38fafad1ec303a62bc9b6dfc9 Mon Sep 17 00:00:00 2001 From: sanaz Date: Mon, 10 Jul 2023 13:13:46 -0700 Subject: [PATCH 35/39] addresses reviewers feedback --- extendeddatacrossword_test.go | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index a63a1ac..c9d7d54 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -346,20 +346,20 @@ func createTestEds(codec Codec, shareSize int) *ExtendedDataSquare { return eds } -func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { - shareSize := 64 +func TestCorruptedEdsReturnsErrByzantineData_UnorderedShares(t *testing.T) { + shareSize := 512 namespaceSize := 1 one := bytes.Repeat([]byte{1}, shareSize) two := bytes.Repeat([]byte{2}, shareSize) three := bytes.Repeat([]byte{3}, shareSize) sharesValue := []int{1, 2, 3, 4} tests := []struct { - name string - coords [][]uint - values [][]byte - wantErr bool - corruptedAxis Axis - corruptedInd uint + name string + coords [][]uint + values [][]byte + wantErr bool + corruptedAxis Axis + corruptedIndex uint }{ { name: "no corruption", @@ -392,7 +392,7 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, nil, }, - corruptedInd: 0, + corruptedIndex: 0, }, { // disturbs the order of shares in the first column, erases the rest of the eds @@ -421,7 +421,7 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { nil, nil, nil, nil, nil, nil, }, - corruptedInd: 0, + corruptedIndex: 0, }, } @@ -439,7 +439,7 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { t.Run(test.name, func(t *testing.T) { // create an eds with the given shares corruptEds := createTestEdsWithNMT(t, codec, shareSize, namespaceSize, sharesValue...) - assert.NotNil(t, eds) + assert.NotNil(t, corruptEds) // corrupt it by setting the values at the given coordinates for i, coords := range test.coords { x := coords[0] @@ -454,7 +454,7 @@ func TestCorruptedEdsReturnsErrByzantineData_UorderedShares(t *testing.T) { assert.ErrorAs(t, err, &byzErr) errors.As(err, &byzErr) assert.Equal(t, byzErr.Axis, test.corruptedAxis) - assert.Equal(t, byzErr.Index, test.corruptedInd) + assert.Equal(t, byzErr.Index, test.corruptedIndex) } }) } @@ -479,9 +479,7 @@ func createTestEdsWithNMT(t *testing.T, codec Codec, shareSize, namespaceSize in odsWidth := edsWidth / 2 // number of shares per row/column in the original data square eds, err := ComputeExtendedDataSquare(shares, codec, newConstructor(uint64(odsWidth), nmt.NamespaceIDSize(namespaceSize))) - if err != nil { - panic(err) - } + require.NoError(t, err) return eds } From 1a4e055a9427c3e37e28dc72d22922d4dc754443 Mon Sep 17 00:00:00 2001 From: sanaz Date: Mon, 10 Jul 2023 13:15:40 -0700 Subject: [PATCH 36/39] fixes a variable name --- extendeddatasquare_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extendeddatasquare_test.go b/extendeddatasquare_test.go index 2b47ab3..a81c247 100644 --- a/extendeddatasquare_test.go +++ b/extendeddatasquare_test.go @@ -74,7 +74,7 @@ func TestComputeExtendedDataSquare(t *testing.T) { func TestImportExtendedDataSquare(t *testing.T) { t.Run("is able to import an EDS", func(t *testing.T) { - eds := createExampleEds(t, ShardSize) + eds := createExampleEds(t, shareSize) got, err := ImportExtendedDataSquare(eds.Flattened(), NewLeoRSCodec(), NewDefaultTree) assert.NoError(t, err) assert.Equal(t, eds.Flattened(), got.Flattened()) From a364d791b546ce08e8f9da5fa926bcd53a0e8f33 Mon Sep 17 00:00:00 2001 From: sanaz Date: Mon, 10 Jul 2023 13:18:16 -0700 Subject: [PATCH 37/39] replaces reference to the nmtwrapper with a permalink --- nmtwrapper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nmtwrapper.go b/nmtwrapper.go index b9b5b92..f5a610e 100644 --- a/nmtwrapper.go +++ b/nmtwrapper.go @@ -1,6 +1,6 @@ package rsmt2d -// The contents of this file have been adapted from the source file available at https://github.com/celestiaorg/celestia-app/blob/main/pkg/wrapper/nmt_wrapper.go, +// The contents of this file have been adapted from the source file available at https://github.com/celestiaorg/celestia-app/blob/bab6c0d0befe677ab8c2f4b83561c08affc7203e/pkg/wrapper/nmt_wrapper.go#L1, // solely for the purpose of testing rsmt2d expected behavior when integrated with a NamespaceMerkleTree. // Please note that this file has undergone several modifications and may not match the original file exactly. From 799ee73e2a8c8e4a26b3f7e2b1cebfb835ace15b Mon Sep 17 00:00:00 2001 From: sanaz Date: Mon, 10 Jul 2023 13:37:20 -0700 Subject: [PATCH 38/39] adds _test suffix to the nmtwrapper file --- nmtwrapper.go => nmtwrapper_test.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename nmtwrapper.go => nmtwrapper_test.go (100%) diff --git a/nmtwrapper.go b/nmtwrapper_test.go similarity index 100% rename from nmtwrapper.go rename to nmtwrapper_test.go From 1e7b68f01e58ab4a5078841d4ff016e64f7f94c2 Mon Sep 17 00:00:00 2001 From: sanaz Date: Tue, 11 Jul 2023 10:12:21 -0700 Subject: [PATCH 39/39] removes line number from the permalink --- nmtwrapper_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nmtwrapper_test.go b/nmtwrapper_test.go index f5a610e..a651fc2 100644 --- a/nmtwrapper_test.go +++ b/nmtwrapper_test.go @@ -1,6 +1,6 @@ package rsmt2d -// The contents of this file have been adapted from the source file available at https://github.com/celestiaorg/celestia-app/blob/bab6c0d0befe677ab8c2f4b83561c08affc7203e/pkg/wrapper/nmt_wrapper.go#L1, +// The contents of this file have been adapted from the source file available at https://github.com/celestiaorg/celestia-app/blob/bab6c0d0befe677ab8c2f4b83561c08affc7203e/pkg/wrapper/nmt_wrapper.go, // solely for the purpose of testing rsmt2d expected behavior when integrated with a NamespaceMerkleTree. // Please note that this file has undergone several modifications and may not match the original file exactly.