Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: chunk -> share #306

Merged
merged 4 commits into from
Mar 11, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions codec_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,10 @@ func BenchmarkEncoding(b *testing.B) {
}
}

func generateRandData(count int, chunkSize int) [][]byte {
func generateRandData(count int, shareSize int) [][]byte {
out := make([][]byte, count)
for i := 0; i < count; i++ {
randData := make([]byte, chunkSize)
randData := make([]byte, shareSize)
_, err := cryptorand.Read(randData)
if err != nil {
panic(err)
Expand Down Expand Up @@ -70,8 +70,8 @@ func BenchmarkDecoding(b *testing.B) {
}
}

func generateMissingData(count int, chunkSize int, codec Codec) [][]byte {
randData := generateRandData(count, chunkSize)
func generateMissingData(count int, shareSize int, codec Codec) [][]byte {
randData := generateRandData(count, shareSize)
encoded, err := codec.Encode(randData)
if err != nil {
panic(err)
Expand All @@ -98,12 +98,12 @@ func newTestCodec() Codec {
return &testCodec{}
}

func (c *testCodec) Encode(chunk [][]byte) ([][]byte, error) {
return chunk, nil
func (c *testCodec) Encode(share [][]byte) ([][]byte, error) {
return share, nil
}

func (c *testCodec) Decode(chunk [][]byte) ([][]byte, error) {
return chunk, nil
func (c *testCodec) Decode(share [][]byte) ([][]byte, error) {
return share, nil
}

func (c *testCodec) MaxChunks() int {
Expand Down
9 changes: 5 additions & 4 deletions codecs.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ const (
// Leopard is a codec that was originally implemented in the C++ library
// https://github.com/catid/leopard. rsmt2d uses a Go port of the C++
// library in https://github.com/klauspost/reedsolomon. The Leopard codec
// uses 8-bit leopard for shards less than or equal to 256. The Leopard
// codec uses 16-bit leopard for shards greater than 256.
// uses 8-bit leopard for shares less than or equal to 256. The Leopard
// codec uses 16-bit leopard for shares greater than 256.
Leopard = "Leopard"
)

Expand All @@ -19,12 +19,13 @@ type Codec interface {
// Missing shares must be nil. Returns original + parity data.
Decode(data [][]byte) ([][]byte, error)
// MaxChunks returns the max number of chunks this codec supports in a 2D
// original data square.
// original data square. Chunk is a synonym of share.
MaxChunks() int
rootulp marked this conversation as resolved.
Show resolved Hide resolved
// Name returns the name of the codec.
Name() string
// ValidateChunkSize returns an error if this codec does not support
// chunkSize. Returns nil if chunkSize is supported.
// chunkSize. Returns nil if chunkSize is supported. Chunk is a synonym of
// share.
ValidateChunkSize(chunkSize int) error
}

Expand Down
48 changes: 27 additions & 21 deletions datasquare.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@ import (
"golang.org/x/sync/errgroup"
)

// ErrUnevenChunks is thrown when non-nil chunks are not all of equal size.
var ErrUnevenChunks = errors.New("non-nil chunks not all of equal size")
// ErrUnevenChunks is thrown when non-nil shares are not all of equal size.
// Note: chunks is synonymous with shares.
var ErrUnevenChunks = errors.New("non-nil shares not all of equal size")
staheri14 marked this conversation as resolved.
Show resolved Hide resolved

// dataSquare stores all data for an original data square (ODS) or extended
// data square (EDS). Data is duplicated in both row-major and column-major
Expand All @@ -20,7 +21,7 @@ type dataSquare struct {
squareCol [][][]byte // col-major
dataMutex sync.Mutex
width uint
chunkSize uint
shareSize uint
rowRoots [][]byte
colRoots [][]byte
createTreeFn TreeConstructorFn
Expand All @@ -29,14 +30,15 @@ type dataSquare struct {
// newDataSquare populates the data square from the supplied data and treeCreator.
// No root calculation is performed.
// data may have nil values.
func newDataSquare(data [][]byte, treeCreator TreeConstructorFn, chunkSize uint) (*dataSquare, error) {
func newDataSquare(data [][]byte, treeCreator TreeConstructorFn, shareSize uint) (*dataSquare, error) {
width := int(math.Ceil(math.Sqrt(float64(len(data)))))
if width*width != len(data) {
// TODO: export this error and modify chunks to shares
return nil, errors.New("number of chunks must be a square number")
}

for _, d := range data {
if d != nil && len(d) != int(chunkSize) {
if d != nil && len(d) != int(shareSize) {
return nil, ErrUnevenChunks
}
}
Expand All @@ -46,7 +48,7 @@ func newDataSquare(data [][]byte, treeCreator TreeConstructorFn, chunkSize uint)
squareRow[i] = data[i*width : i*width+width]

for j := 0; j < width; j++ {
if squareRow[i][j] != nil && len(squareRow[i][j]) != int(chunkSize) {
if squareRow[i][j] != nil && len(squareRow[i][j]) != int(shareSize) {
return nil, ErrUnevenChunks
}
}
Expand All @@ -64,15 +66,16 @@ func newDataSquare(data [][]byte, treeCreator TreeConstructorFn, chunkSize uint)
squareRow: squareRow,
squareCol: squareCol,
width: uint(width),
chunkSize: chunkSize,
shareSize: shareSize,
createTreeFn: treeCreator,
}, nil
}

// extendSquare extends the original data square by extendedWidth and fills
// the extended quadrants with fillerChunk.
func (ds *dataSquare) extendSquare(extendedWidth uint, fillerChunk []byte) error {
if uint(len(fillerChunk)) != ds.chunkSize {
// the extended quadrants with fillerShare.
func (ds *dataSquare) extendSquare(extendedWidth uint, fillerShare []byte) error {
if uint(len(fillerShare)) != ds.shareSize {
// TODO: export this error and rename chunk to share
return errors.New("filler chunk size does not match data square chunk size")
}

Expand All @@ -81,12 +84,12 @@ func (ds *dataSquare) extendSquare(extendedWidth uint, fillerChunk []byte) error

fillerExtendedRow := make([][]byte, extendedWidth)
for i := uint(0); i < extendedWidth; i++ {
fillerExtendedRow[i] = fillerChunk
fillerExtendedRow[i] = fillerShare
}

fillerRow := make([][]byte, newWidth)
for i := uint(0); i < newWidth; i++ {
fillerRow[i] = fillerChunk
fillerRow[i] = fillerShare
}

row := make([][]byte, ds.width)
Expand Down Expand Up @@ -129,7 +132,8 @@ func (ds *dataSquare) row(x uint) [][]byte {

func (ds *dataSquare) setRowSlice(x uint, y uint, newRow [][]byte) error {
for i := uint(0); i < uint(len(newRow)); i++ {
if len(newRow[i]) != int(ds.chunkSize) {
if len(newRow[i]) != int(ds.shareSize) {
// TODO: export this error and rename chunk to share
return errors.New("invalid chunk size")
}
}
Expand Down Expand Up @@ -162,7 +166,8 @@ func (ds *dataSquare) col(y uint) [][]byte {

func (ds *dataSquare) setColSlice(x uint, y uint, newCol [][]byte) error {
for i := uint(0); i < uint(len(newCol)); i++ {
if len(newCol[i]) != int(ds.chunkSize) {
if len(newCol[i]) != int(ds.shareSize) {
// TODO: export this error and rename chunk to share
return errors.New("invalid chunk size")
}
}
Expand Down Expand Up @@ -307,22 +312,23 @@ func (ds *dataSquare) GetCell(x uint, y uint) []byte {
if ds.squareRow[x][y] == nil {
return nil
}
cell := make([]byte, ds.chunkSize)
cell := make([]byte, ds.shareSize)
copy(cell, ds.squareRow[x][y])
return cell
}

// SetCell sets a specific cell. The cell to set must be `nil`. Returns an error
// if the cell to set is not `nil` or newChunk is not the correct size.
func (ds *dataSquare) SetCell(x uint, y uint, newChunk []byte) error {
// if the cell to set is not `nil` or newShare is not the correct size.
func (ds *dataSquare) SetCell(x uint, y uint, newShare []byte) error {
if ds.squareRow[x][y] != nil {
return fmt.Errorf("cannot set cell (%d, %d) as it already has a value %x", x, y, ds.squareRow[x][y])
}
if len(newChunk) != int(ds.chunkSize) {
return fmt.Errorf("cannot set cell with chunk size %d because dataSquare chunk size is %d", len(newChunk), ds.chunkSize)
if len(newShare) != int(ds.shareSize) {
// TODO: export this error and rename chunk to share
return fmt.Errorf("cannot set cell with chunk size %d because dataSquare chunk size is %d", len(newShare), ds.shareSize)
}
ds.squareRow[x][y] = newChunk
ds.squareCol[y][x] = newChunk
ds.squareRow[x][y] = newShare
ds.squareCol[y][x] = newShare
ds.resetRoots()
return nil
}
Expand Down
32 changes: 16 additions & 16 deletions datasquare_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,15 @@ func TestNewDataSquare(t *testing.T) {
name string
cells [][]byte
expected [][][]byte
chunkSize uint
shareSize uint
}{
{"1x1", [][]byte{{1, 2}}, [][][]byte{{{1, 2}}}, 2},
{"2x2", [][]byte{{1, 2}, {3, 4}, {5, 6}, {7, 8}}, [][][]byte{{{1, 2}, {3, 4}}, {{5, 6}, {7, 8}}}, 2},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
result, err := newDataSquare(test.cells, NewDefaultTree, test.chunkSize)
result, err := newDataSquare(test.cells, NewDefaultTree, test.shareSize)
if err != nil {
panic(err)
}
Expand All @@ -38,16 +38,16 @@ func TestInvalidDataSquareCreation(t *testing.T) {
tests := []struct {
name string
cells [][]byte
chunkSize uint
shareSize uint
}{
{"InconsistentChunkNumber", [][]byte{{1, 2}, {3, 4}, {5, 6}}, 2},
{"UnequalChunkSize", [][]byte{{1, 2}, {3, 4}, {5, 6}, {7}}, 2},
{"InconsistentShareNumber", [][]byte{{1, 2}, {3, 4}, {5, 6}}, 2},
{"UnequalShareSize", [][]byte{{1, 2}, {3, 4}, {5, 6}, {7}}, 2},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
_, err := newDataSquare(test.cells, NewDefaultTree, test.chunkSize)
_, err := newDataSquare(test.cells, NewDefaultTree, test.shareSize)
if err == nil {
t.Errorf("newDataSquare failed; chunks accepted with %v", test.name)
t.Errorf("newDataSquare failed; shares accepted with %v", test.name)
}
})
}
Expand All @@ -74,9 +74,9 @@ func TestSetCell(t *testing.T) {
wantErr: true,
},
{
name: "expect error if new cell is not the correct chunk size",
name: "expect error if new cell is not the correct share size",
originalCell: nil,
newCell: []byte{1, 2}, // incorrect chunk size
newCell: []byte{1, 2}, // incorrect share size
wantErr: true,
},
}
Expand Down Expand Up @@ -184,7 +184,7 @@ func TestInvalidSquareExtension(t *testing.T) {
}
err = ds.extendSquare(1, []byte{0})
if err == nil {
t.Errorf("extendSquare failed; error not returned when filler chunk size does not match data square chunk size")
t.Errorf("extendSquare failed; error not returned when filler share size does not match data square share size")
}
}

Expand Down Expand Up @@ -316,7 +316,7 @@ func Test_setRowSlice(t *testing.T) {
wantErr: false,
},
{
name: "returns an error if the new row has an invalid chunk size",
name: "returns an error if the new row has an invalid share size",
newRow: [][]byte{{5, 6}},
x: 0,
y: 0,
Expand Down Expand Up @@ -372,7 +372,7 @@ func Test_setColSlice(t *testing.T) {
wantErr: false,
},
{
name: "returns an error if the new col has an invalid chunk size",
name: "returns an error if the new col has an invalid share size",
newCol: [][]byte{{5, 6}},
x: 0,
y: 0,
Expand Down Expand Up @@ -408,7 +408,7 @@ func BenchmarkEDSRoots(b *testing.B) {
b.Errorf("Failure to create square of size %d: %s", i, err)
}
b.Run(
fmt.Sprintf("%dx%dx%d ODS", i, i, int(square.chunkSize)),
fmt.Sprintf("%dx%dx%d ODS", i, i, int(square.shareSize)),
func(b *testing.B) {
for n := 0; n < b.N; n++ {
square.resetRoots()
Expand Down Expand Up @@ -470,8 +470,8 @@ func (d *errorTree) Root() ([]byte, error) {
// setCell overwrites the contents of a specific cell. setCell does not perform
// any input validation so most use cases should use `SetCell` instead of
// `setCell`. This method exists strictly for testing.
func (ds *dataSquare) setCell(x uint, y uint, newChunk []byte) {
ds.squareRow[x][y] = newChunk
ds.squareCol[y][x] = newChunk
func (ds *dataSquare) setCell(x uint, y uint, newShare []byte) {
ds.squareRow[x][y] = newShare
ds.squareCol[y][x] = newShare
ds.resetRoots()
}
6 changes: 3 additions & 3 deletions extendeddatacrossword.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func (a Axis) String() string {
}
}

// ErrUnrepairableDataSquare is thrown when there is insufficient chunks to repair the square.
// ErrUnrepairableDataSquare is thrown when there is insufficient shares to repair the square.
var ErrUnrepairableDataSquare = errors.New("failed to solve data square")

// ErrByzantineData is returned when a repaired row or column does not match the
Expand Down Expand Up @@ -380,7 +380,7 @@ func (eds *ExtendedDataSquare) preRepairSanityCheck(
if err != nil {
return err
}
if !bytes.Equal(flattenChunks(parityShares), flattenChunks(eds.rowSlice(i, eds.originalDataWidth, eds.originalDataWidth))) {
if !bytes.Equal(flattenShares(parityShares), flattenShares(eds.rowSlice(i, eds.originalDataWidth, eds.originalDataWidth))) {
return &ErrByzantineData{Row, i, eds.row(i)}
}
return nil
Expand Down Expand Up @@ -410,7 +410,7 @@ func (eds *ExtendedDataSquare) preRepairSanityCheck(
if err != nil {
return err
}
if !bytes.Equal(flattenChunks(parityShares), flattenChunks(eds.colSlice(eds.originalDataWidth, i, eds.originalDataWidth))) {
if !bytes.Equal(flattenShares(parityShares), flattenShares(eds.colSlice(eds.originalDataWidth, i, eds.originalDataWidth))) {
return &ErrByzantineData{Col, i, eds.col(i)}
}
return nil
Expand Down
Loading
Loading