diff --git a/api/docgen/examples.go b/api/docgen/examples.go index 80a8c64d93..3456880c4f 100644 --- a/api/docgen/examples.go +++ b/api/docgen/examples.go @@ -133,13 +133,13 @@ func init() { } addToExampleValues(addrInfo) - namespace, err := share.NewNamespaceV0([]byte{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0x10}) + namespace, err := share.NewBlobNamespaceV0([]byte{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0x10}) if err != nil { panic(err) } addToExampleValues(namespace) - generatedBlob, err := blob.NewBlob(0, namespace, []byte("This is an example of some blob data")) + generatedBlob, err := blob.NewBlobV0(namespace, []byte("This is an example of some blob data")) if err != nil { panic(err) } diff --git a/api/gateway/endpoints.go b/api/gateway/endpoints.go index 0ae93b112c..9600138909 100644 --- a/api/gateway/endpoints.go +++ b/api/gateway/endpoints.go @@ -33,13 +33,13 @@ func (h *Handler) RegisterEndpoints(rpc *Server, deprecatedEndpointsEnabled bool rpc.RegisterHandlerFunc(submitTxEndpoint, h.handleSubmitTx, http.MethodPost) // share endpoints - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}/height/{%s}", namespacedSharesEndpoint, nIDKey, heightKey), + rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}/height/{%s}", namespacedSharesEndpoint, namespaceKey, heightKey), h.handleSharesByNamespaceRequest, http.MethodGet) - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", namespacedSharesEndpoint, nIDKey), + rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", namespacedSharesEndpoint, namespaceKey), h.handleSharesByNamespaceRequest, http.MethodGet) - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}/height/{%s}", namespacedDataEndpoint, nIDKey, heightKey), + rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}/height/{%s}", namespacedDataEndpoint, namespaceKey, heightKey), h.handleDataByNamespaceRequest, http.MethodGet) - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", namespacedDataEndpoint, nIDKey), + rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", namespacedDataEndpoint, namespaceKey), h.handleDataByNamespaceRequest, http.MethodGet) // DAS endpoints diff --git a/api/gateway/share.go b/api/gateway/share.go index db5ed37286..c9dec071f3 100644 --- a/api/gateway/share.go +++ b/api/gateway/share.go @@ -10,7 +10,6 @@ import ( "github.com/gorilla/mux" "github.com/celestiaorg/celestia-app/pkg/shares" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/celestia-node/share" ) @@ -20,7 +19,7 @@ const ( namespacedDataEndpoint = "/namespaced_data" ) -var nIDKey = "nid" +var namespaceKey = "nid" // NamespacedSharesResponse represents the response to a // SharesByNamespace request. @@ -37,12 +36,12 @@ type NamespacedDataResponse struct { } func (h *Handler) handleSharesByNamespaceRequest(w http.ResponseWriter, r *http.Request) { - height, nID, err := parseGetByNamespaceArgs(r) + height, namespace, err := parseGetByNamespaceArgs(r) if err != nil { writeError(w, http.StatusBadRequest, namespacedSharesEndpoint, err) return } - shares, err := h.getShares(r.Context(), height, nID) + shares, err := h.getShares(r.Context(), height, namespace) if err != nil { writeError(w, http.StatusInternalServerError, namespacedSharesEndpoint, err) return @@ -62,12 +61,12 @@ func (h *Handler) handleSharesByNamespaceRequest(w http.ResponseWriter, r *http. } func (h *Handler) handleDataByNamespaceRequest(w http.ResponseWriter, r *http.Request) { - height, nID, err := parseGetByNamespaceArgs(r) + height, namespace, err := parseGetByNamespaceArgs(r) if err != nil { writeError(w, http.StatusBadRequest, namespacedDataEndpoint, err) return } - shares, err := h.getShares(r.Context(), height, nID) + shares, err := h.getShares(r.Context(), height, namespace) if err != nil { writeError(w, http.StatusInternalServerError, namespacedDataEndpoint, err) return @@ -91,13 +90,13 @@ func (h *Handler) handleDataByNamespaceRequest(w http.ResponseWriter, r *http.Re } } -func (h *Handler) getShares(ctx context.Context, height uint64, nID namespace.ID) ([]share.Share, error) { +func (h *Handler) getShares(ctx context.Context, height uint64, namespace share.Namespace) ([]share.Share, error) { header, err := h.header.GetByHeight(ctx, height) if err != nil { return nil, err } - shares, err := h.share.GetSharesByNamespace(ctx, header.DAH, nID) + shares, err := h.share.GetSharesByNamespace(ctx, header.DAH, namespace) if err != nil { return nil, err } @@ -124,7 +123,7 @@ func dataFromShares(input []share.Share) (data [][]byte, err error) { return data, nil } -func parseGetByNamespaceArgs(r *http.Request) (height uint64, nID namespace.ID, err error) { +func parseGetByNamespaceArgs(r *http.Request) (height uint64, namespace share.Namespace, err error) { vars := mux.Vars(r) // if a height was given, parse it, otherwise get namespaced shares/data from the latest header if strHeight, ok := vars[heightKey]; ok { @@ -133,11 +132,10 @@ func parseGetByNamespaceArgs(r *http.Request) (height uint64, nID namespace.ID, return 0, nil, err } } - hexNID := vars[nIDKey] - nID, err = hex.DecodeString(hexNID) + hexNamespace := vars[namespaceKey] + namespace, err = hex.DecodeString(hexNamespace) if err != nil { return 0, nil, err } - - return height, nID, nil + return height, namespace, namespace.ValidateForData() } diff --git a/api/gateway/share_test.go b/api/gateway/share_test.go index 16cf606680..9b12240f62 100644 --- a/api/gateway/share_test.go +++ b/api/gateway/share_test.go @@ -8,8 +8,9 @@ import ( coretypes "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-app/pkg/appconsts" - "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" + + "github.com/celestiaorg/celestia-node/share/sharetest" ) func Test_dataFromShares(t *testing.T) { @@ -19,13 +20,13 @@ func Test_dataFromShares(t *testing.T) { []byte("BEEEEAHP"), } - ns := namespace.RandomBlobNamespace() + ns := sharetest.RandV0Namespace() sss := shares.NewSparseShareSplitter() for _, data := range testData { b := coretypes.Blob{ Data: data, - NamespaceID: ns.ID, - NamespaceVersion: ns.Version, + NamespaceID: ns.ID(), + NamespaceVersion: ns.Version(), ShareVersion: appconsts.ShareVersionZero, } err := sss.Write(b) diff --git a/api/gateway/state.go b/api/gateway/state.go index b584b00d36..69900b0bfc 100644 --- a/api/gateway/state.go +++ b/api/gateway/state.go @@ -9,8 +9,6 @@ import ( "github.com/cosmos/cosmos-sdk/types" "github.com/gorilla/mux" - "github.com/celestiaorg/celestia-app/pkg/appconsts" - "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/state" ) @@ -131,7 +129,7 @@ func (h *Handler) handleSubmitPFB(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, submitPFBEndpoint, err) return } - nID, err := hex.DecodeString(req.NamespaceID) + namespace, err := hex.DecodeString(req.NamespaceID) if err != nil { writeError(w, http.StatusBadRequest, submitPFBEndpoint, err) return @@ -143,17 +141,22 @@ func (h *Handler) handleSubmitPFB(w http.ResponseWriter, r *http.Request) { } fee := types.NewInt(req.Fee) - constructedBlob, err := blob.NewBlob(appconsts.DefaultShareVersion, nID, data) + constructedBlob, err := blob.NewBlobV0(namespace, data) if err != nil { writeError(w, http.StatusBadRequest, submitPFBEndpoint, err) return } // perform request - txResp, txerr := h.state.SubmitPayForBlob(r.Context(), fee, req.GasLimit, []*blob.Blob{constructedBlob}) - if txerr != nil && txResp == nil { - // no tx data to return - writeError(w, http.StatusInternalServerError, submitPFBEndpoint, err) + txResp, err := h.state.SubmitPayForBlob(r.Context(), fee, req.GasLimit, []*blob.Blob{constructedBlob}) + if err != nil { + if txResp == nil { + // no tx data to return + writeError(w, http.StatusBadRequest, submitPFBEndpoint, err) + return + } + // if error returned, change status from 200 to 206 + w.WriteHeader(http.StatusPartialContent) } bs, err := json.Marshal(&txResp) @@ -162,10 +165,6 @@ func (h *Handler) handleSubmitPFB(w http.ResponseWriter, r *http.Request) { return } - // if error returned, change status from 200 to 206 - if txerr != nil { - w.WriteHeader(http.StatusPartialContent) - } _, err = w.Write(bs) if err != nil { log.Errorw("writing response", "endpoint", submitPFBEndpoint, "err", err) diff --git a/api/gateway/state_test.go b/api/gateway/state_test.go index a613471a04..aa9196cc8d 100644 --- a/api/gateway/state_test.go +++ b/api/gateway/state_test.go @@ -34,7 +34,7 @@ func TestHandleSubmitPFB(t *testing.T) { mock.EXPECT().SubmitPayForBlob(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(&txResponse, timedErr) - ns, err := share.NewNamespaceV0([]byte("abc")) + ns, err := share.NewBlobNamespaceV0([]byte("abc")) require.NoError(t, err) hexNs := hex.EncodeToString(ns[:]) diff --git a/blob/blob.go b/blob/blob.go index 9771714cb9..e9ad2b6255 100644 --- a/blob/blob.go +++ b/blob/blob.go @@ -5,11 +5,13 @@ import ( "encoding/json" "fmt" - appns "github.com/celestiaorg/celestia-app/pkg/namespace" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/nmt" - "github.com/celestiaorg/nmt/namespace" + "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/ipld" ) @@ -101,41 +103,51 @@ type Blob struct { types.Blob `json:"blob"` Commitment Commitment `json:"commitment"` + + // the celestia-node's namespace type + // this is to avoid converting to and from app's type + namespace share.Namespace } -// NewBlob constructs a new blob from the provided namespace.ID and data. -func NewBlob(shareVersion uint8, namespace namespace.ID, data []byte) (*Blob, error) { - if len(namespace) != appns.NamespaceSize { - return nil, fmt.Errorf("invalid size of the namespace id. got:%d, want:%d", len(namespace), appns.NamespaceSize) - } +// NewBlobV0 constructs a new blob from the provided Namespace and data. +// The blob will be formatted as v0 shares. +func NewBlobV0(namespace share.Namespace, data []byte) (*Blob, error) { + return NewBlob(appconsts.ShareVersionZero, namespace, data) +} - ns, err := appns.New(namespace[appns.NamespaceVersionSize-1], namespace[appns.NamespaceVersionSize:]) - if err != nil { +// NewBlob constructs a new blob from the provided Namespace, data and share version. +func NewBlob(shareVersion uint8, namespace share.Namespace, data []byte) (*Blob, error) { + if len(data) == 0 || len(data) > appconsts.DefaultMaxBytes { + return nil, fmt.Errorf("blob data must be > 0 && <= %d, but it was %d bytes", appconsts.DefaultMaxBytes, len(data)) + } + if err := namespace.ValidateForBlob(); err != nil { return nil, err } - blob, err := types.NewBlob(ns, data, shareVersion) - if err != nil { - return nil, err + blob := tmproto.Blob{ + NamespaceId: namespace.ID(), + Data: data, + ShareVersion: uint32(shareVersion), + NamespaceVersion: uint32(namespace.Version()), } - com, err := types.CreateCommitment(blob) + com, err := types.CreateCommitment(&blob) if err != nil { return nil, err } - return &Blob{Blob: *blob, Commitment: com}, nil + return &Blob{Blob: blob, Commitment: com, namespace: namespace}, nil } // Namespace returns blob's namespace. -func (b *Blob) Namespace() namespace.ID { - return append([]byte{uint8(b.NamespaceVersion)}, b.NamespaceId...) +func (b *Blob) Namespace() share.Namespace { + return b.namespace } type jsonBlob struct { - Namespace namespace.ID `json:"namespace"` - Data []byte `json:"data"` - ShareVersion uint32 `json:"share_version"` - Commitment Commitment `json:"commitment"` + Namespace share.Namespace `json:"namespace"` + Data []byte `json:"data"` + ShareVersion uint32 `json:"share_version"` + Commitment Commitment `json:"commitment"` } func (b *Blob) MarshalJSON() ([]byte, error) { @@ -155,10 +167,11 @@ func (b *Blob) UnmarshalJSON(data []byte) error { return err } - b.Blob.NamespaceVersion = uint32(blob.Namespace[0]) - b.Blob.NamespaceId = blob.Namespace[1:] + b.Blob.NamespaceVersion = uint32(blob.Namespace.Version()) + b.Blob.NamespaceId = blob.Namespace.ID() b.Blob.Data = blob.Data b.Blob.ShareVersion = blob.ShareVersion b.Commitment = blob.Commitment + b.namespace = blob.Namespace return nil } diff --git a/blob/blob_test.go b/blob/blob_test.go index 3aabd6559b..85486ad125 100644 --- a/blob/blob_test.go +++ b/blob/blob_test.go @@ -8,14 +8,13 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/types" - appns "github.com/celestiaorg/celestia-app/pkg/namespace" apptypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/celestia-node/blob/blobtest" ) func TestBlob(t *testing.T) { - appBlobs, err := blobtest.GenerateBlobs([]int{1}, false) + appBlobs, err := blobtest.GenerateV0Blobs([]int{1}, false) require.NoError(t, err) blob, err := convertBlobs(appBlobs...) require.NoError(t, err) @@ -42,12 +41,9 @@ func TestBlob(t *testing.T) { }, }, { - name: "verify nID", + name: "verify namespace", expectedRes: func(t *testing.T) { - ns, err := appns.New( - blob[0].Namespace()[appns.NamespaceVersionSize-1], - blob[0].Namespace()[appns.NamespaceVersionSize:], - ) + ns := blob[0].Namespace().ToAppNamespace() require.NoError(t, err) require.NoError(t, apptypes.ValidateBlobNamespace(ns)) }, diff --git a/blob/blobtest/testing.go b/blob/blobtest/testing.go index 395ef4167a..a22f22f790 100644 --- a/blob/blobtest/testing.go +++ b/blob/blobtest/testing.go @@ -11,14 +11,16 @@ import ( "github.com/celestiaorg/celestia-node/share" ) -func GenerateBlobs(sizes []int, sameNID bool) ([]types.Blob, error) { +// GenerateV0Blobs is a test utility producing v0 share formatted blobs with the +// requested size and random namespaces. +func GenerateV0Blobs(sizes []int, sameNamespace bool) ([]types.Blob, error) { blobs := make([]types.Blob, 0, len(sizes)) for _, size := range sizes { size := rawBlobSize(appconsts.FirstSparseShareContentSize * size) appBlob := testfactory.GenerateRandomBlob(size) - if !sameNID { - nid, err := share.NewNamespaceV0(tmrand.Bytes(7)) + if !sameNamespace { + nid, err := share.NewBlobNamespaceV0(tmrand.Bytes(7)) if err != nil { return nil, err } diff --git a/blob/helper.go b/blob/helper.go index 1fef41dc22..341080f42f 100644 --- a/blob/helper.go +++ b/blob/helper.go @@ -19,8 +19,8 @@ func SharesToBlobs(rawShares []share.Share) ([]*Blob, error) { } appShares := make([]shares.Share, 0, len(rawShares)) - for _, sh := range rawShares { - bShare, err := shares.NewShare(sh) + for _, shr := range rawShares { + bShare, err := shares.NewShare(shr) if err != nil { return nil, err } diff --git a/blob/service.go b/blob/service.go index 0876a45a0c..960f8c6c01 100644 --- a/blob/service.go +++ b/blob/service.go @@ -12,7 +12,6 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/shares" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" @@ -72,8 +71,8 @@ func (s *Service) Submit(ctx context.Context, blobs []*Blob) (uint64, error) { } // Get retrieves all the blobs for given namespaces at the given height by commitment. -func (s *Service) Get(ctx context.Context, height uint64, nID namespace.ID, commitment Commitment) (*Blob, error) { - blob, _, err := s.getByCommitment(ctx, height, nID, commitment) +func (s *Service) Get(ctx context.Context, height uint64, ns share.Namespace, commitment Commitment) (*Blob, error) { + blob, _, err := s.getByCommitment(ctx, height, ns, commitment) if err != nil { return nil, err } @@ -85,10 +84,10 @@ func (s *Service) Get(ctx context.Context, height uint64, nID namespace.ID, comm func (s *Service) GetProof( ctx context.Context, height uint64, - nID namespace.ID, + namespace share.Namespace, commitment Commitment, ) (*Proof, error) { - _, proof, err := s.getByCommitment(ctx, height, nID, commitment) + _, proof, err := s.getByCommitment(ctx, height, namespace, commitment) if err != nil { return nil, err } @@ -97,29 +96,29 @@ func (s *Service) GetProof( // GetAll returns all blobs under the given namespaces at the given height. // GetAll can return blobs and an error in case if some requests failed. -func (s *Service) GetAll(ctx context.Context, height uint64, nIDs []namespace.ID) ([]*Blob, error) { +func (s *Service) GetAll(ctx context.Context, height uint64, namespaces []share.Namespace) ([]*Blob, error) { header, err := s.headerGetter(ctx, height) if err != nil { return nil, err } var ( - resultBlobs = make([][]*Blob, len(nIDs)) - resultErr = make([]error, len(nIDs)) + resultBlobs = make([][]*Blob, len(namespaces)) + resultErr = make([]error, len(namespaces)) ) wg := sync.WaitGroup{} - for i, nID := range nIDs { + for i, namespace := range namespaces { wg.Add(1) - go func(i int, nID namespace.ID) { + go func(i int, namespace share.Namespace) { defer wg.Done() - blobs, err := s.getBlobs(ctx, nID, header.DAH) + blobs, err := s.getBlobs(ctx, namespace, header.DAH) if err != nil { - resultErr[i] = fmt.Errorf("getting blobs for nID(%s): %s", nID.String(), err) + resultErr[i] = fmt.Errorf("getting blobs for namespace(%s): %s", namespace.String(), err) return } resultBlobs[i] = blobs - }(i, nID) + }(i, namespace) } wg.Wait() @@ -143,7 +142,7 @@ func (s *Service) GetAll(ctx context.Context, height uint64, nIDs []namespace.ID func (s *Service) Included( ctx context.Context, height uint64, - nID namespace.ID, + namespace share.Namespace, proof *Proof, com Commitment, ) (bool, error) { @@ -156,7 +155,7 @@ func (s *Service) Included( // but we have to guarantee that all our stored subtree roots will be on the same height(e.g. one // level above shares). // TODO(@vgonkivs): rework the implementation to perform all verification without network requests. - _, resProof, err := s.getByCommitment(ctx, height, nID, com) + _, resProof, err := s.getByCommitment(ctx, height, namespace, com) switch err { case nil: case ErrBlobNotFound: @@ -172,12 +171,12 @@ func (s *Service) Included( func (s *Service) getByCommitment( ctx context.Context, height uint64, - nID namespace.ID, + namespace share.Namespace, commitment Commitment, ) (*Blob, *Proof, error) { log.Infow("requesting blob", "height", height, - "nID", nID.String()) + "namespace", namespace.String()) header, err := s.headerGetter(ctx, height) if err != nil { @@ -191,7 +190,7 @@ func (s *Service) getByCommitment( blobShare *shares.Share ) - namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, header.DAH, nID) + namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, header.DAH, namespace) if err != nil { if errors.Is(err, share.ErrNotFound) { err = ErrBlobNotFound @@ -209,8 +208,8 @@ func (s *Service) getByCommitment( // reconstruct the `blobShare` from the first rawShare in range // in order to get blob's length(first share will contain this info) if blobShare == nil { - for i, sh := range rawShares { - bShare, err := shares.NewShare(sh) + for i, shr := range rawShares { + bShare, err := shares.NewShare(shr) if err != nil { return nil, nil, err } @@ -278,10 +277,10 @@ func (s *Service) getByCommitment( return nil, nil, ErrBlobNotFound } -// getBlobs retrieves the DAH and fetches all shares from the requested namespace.ID and converts +// getBlobs retrieves the DAH and fetches all shares from the requested Namespace and converts // them to Blobs. -func (s *Service) getBlobs(ctx context.Context, nID namespace.ID, root *share.Root) ([]*Blob, error) { - namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, root, nID) +func (s *Service) getBlobs(ctx context.Context, namespace share.Namespace, root *share.Root) ([]*Blob, error) { + namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, root, namespace) if err != nil { return nil, err } diff --git a/blob/service_test.go b/blob/service_test.go index ee6e982fe8..51475178d2 100644 --- a/blob/service_test.go +++ b/blob/service_test.go @@ -14,17 +14,15 @@ import ( "github.com/stretchr/testify/require" tmrand "github.com/tendermint/tendermint/libs/rand" - "github.com/celestiaorg/celestia-app/pkg/appconsts" - appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/go-header/store" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/celestia-node/blob/blobtest" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/getters" + "github.com/celestiaorg/celestia-node/share/ipld" ) func TestBlobService_Get(t *testing.T) { @@ -37,12 +35,12 @@ func TestBlobService_Get(t *testing.T) { blobSize3 = 12 ) - appBlobs, err := blobtest.GenerateBlobs([]int{blobSize0, blobSize1}, false) + appBlobs, err := blobtest.GenerateV0Blobs([]int{blobSize0, blobSize1}, false) require.NoError(t, err) blobs0, err := convertBlobs(appBlobs...) require.NoError(t, err) - appBlobs, err = blobtest.GenerateBlobs([]int{blobSize2, blobSize3}, true) + appBlobs, err = blobtest.GenerateV0Blobs([]int{blobSize2, blobSize3}, true) require.NoError(t, err) blobs1, err := convertBlobs(appBlobs...) require.NoError(t, err) @@ -71,9 +69,9 @@ func TestBlobService_Get(t *testing.T) { }, }, { - name: "get all with the same nID", + name: "get all with the same namespace", doFn: func() (interface{}, error) { - b, err := service.GetAll(ctx, 1, []namespace.ID{blobs1[0].Namespace()}) + b, err := service.GetAll(ctx, 1, []share.Namespace{blobs1[0].Namespace()}) return b, err }, expectedResult: func(res interface{}, err error) { @@ -91,9 +89,9 @@ func TestBlobService_Get(t *testing.T) { }, }, { - name: "get all with different nIDs", + name: "get all with different namespaces", doFn: func() (interface{}, error) { - b, err := service.GetAll(ctx, 1, []namespace.ID{blobs0[0].Namespace(), blobs0[1].Namespace()}) + b, err := service.GetAll(ctx, 1, []share.Namespace{blobs0[0].Namespace(), blobs0[1].Namespace()}) return b, err }, expectedResult: func(res interface{}, err error) { @@ -126,7 +124,7 @@ func TestBlobService_Get(t *testing.T) { { name: "get invalid blob", doFn: func() (interface{}, error) { - appBlob, err := blobtest.GenerateBlobs([]int{10}, false) + appBlob, err := blobtest.GenerateV0Blobs([]int{10}, false) require.NoError(t, err) blob, err := convertBlobs(appBlob...) require.NoError(t, err) @@ -157,13 +155,13 @@ func TestBlobService_Get(t *testing.T) { proof, ok := res.(*Proof) assert.True(t, ok) - verifyFn := func(t *testing.T, rawShares [][]byte, proof *Proof, nID namespace.ID) { + verifyFn := func(t *testing.T, rawShares [][]byte, proof *Proof, namespace share.Namespace) { for _, row := range header.DAH.RowRoots { to := 0 for _, p := range *proof { from := to to = p.End() - p.Start() + from - eq := p.VerifyInclusion(sha256.New(), nID, rawShares[from:to], row) + eq := p.VerifyInclusion(sha256.New(), namespace.ToNMT(), rawShares[from:to], row) if eq == true { return } @@ -209,7 +207,7 @@ func TestBlobService_Get(t *testing.T) { { name: "not included", doFn: func() (interface{}, error) { - appBlob, err := blobtest.GenerateBlobs([]int{10}, false) + appBlob, err := blobtest.GenerateV0Blobs([]int{10}, false) require.NoError(t, err) blob, err := convertBlobs(appBlob...) require.NoError(t, err) @@ -256,8 +254,8 @@ func TestBlobService_Get(t *testing.T) { { name: "get all not found", doFn: func() (interface{}, error) { - nID := tmrand.Bytes(appconsts.NamespaceSize) - return service.GetAll(ctx, 1, []namespace.ID{nID}) + namespace := share.Namespace(tmrand.Bytes(share.NamespaceSize)) + return service.GetAll(ctx, 1, []share.Namespace{namespace}) }, expectedResult: func(i interface{}, err error) { blobs, ok := i.([]*Blob) @@ -297,23 +295,19 @@ func TestBlobService_Get(t *testing.T) { } } -// TestService_GetSingleBlobWithoutPadding creates two blobs with the same nID +// TestService_GetSingleBlobWithoutPadding creates two blobs with the same namespace // But to satisfy the rule of eds creating, padding namespace share is placed between // blobs. Test ensures that blob service will skip padding share and return the correct blob. func TestService_GetSingleBlobWithoutPadding(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) t.Cleanup(cancel) - appBlob, err := blobtest.GenerateBlobs([]int{9, 5}, true) + appBlob, err := blobtest.GenerateV0Blobs([]int{9, 5}, true) require.NoError(t, err) blobs, err := convertBlobs(appBlob...) require.NoError(t, err) - ns1, err := appns.New(blobs[0].Namespace()[0], blobs[0].Namespace()[appns.NamespaceVersionSize:]) - require.NoError(t, err) - - ns2, err := appns.New(blobs[1].Namespace()[0], blobs[1].Namespace()[appns.NamespaceVersionSize:]) - require.NoError(t, err) + ns1, ns2 := blobs[0].Namespace().ToAppNamespace(), blobs[1].Namespace().ToAppNamespace() padding0, err := shares.NamespacePaddingShare(ns1) require.NoError(t, err) @@ -332,7 +326,7 @@ func TestService_GetSingleBlobWithoutPadding(t *testing.T) { batching := ds_sync.MutexWrap(ds.NewMapDatastore()) headerStore, err := store.NewStore[*header.ExtendedHeader](batching) require.NoError(t, err) - eds, err := share.AddShares(ctx, rawShares, bs) + eds, err := ipld.AddShares(ctx, rawShares, bs) require.NoError(t, err) h := headertest.ExtendedHeaderFromEDS(t, 1, eds) @@ -353,22 +347,12 @@ func TestService_GetAllWithoutPadding(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) t.Cleanup(cancel) - appBlob, err := blobtest.GenerateBlobs([]int{9, 5}, true) + appBlob, err := blobtest.GenerateV0Blobs([]int{9, 5}, true) require.NoError(t, err) blobs, err := convertBlobs(appBlob...) require.NoError(t, err) - ns1, err := appns.New( - blobs[0].Namespace()[appns.NamespaceVersionSize-1], - blobs[0].Namespace()[appns.NamespaceVersionSize:], - ) - require.NoError(t, err) - - ns2, err := appns.New( - blobs[1].Namespace()[appns.NamespaceVersionSize-1], - blobs[1].Namespace()[appns.NamespaceVersionSize:], - ) - require.NoError(t, err) + ns1, ns2 := blobs[0].Namespace().ToAppNamespace(), blobs[1].Namespace().ToAppNamespace() padding0, err := shares.NamespacePaddingShare(ns1) require.NoError(t, err) @@ -393,7 +377,7 @@ func TestService_GetAllWithoutPadding(t *testing.T) { batching := ds_sync.MutexWrap(ds.NewMapDatastore()) headerStore, err := store.NewStore[*header.ExtendedHeader](batching) require.NoError(t, err) - eds, err := share.AddShares(ctx, rawShares, bs) + eds, err := ipld.AddShares(ctx, rawShares, bs) require.NoError(t, err) h := headertest.ExtendedHeaderFromEDS(t, 1, eds) @@ -406,7 +390,7 @@ func TestService_GetAllWithoutPadding(t *testing.T) { service := NewService(nil, getters.NewIPLDGetter(bs), fn) - _, err = service.GetAll(ctx, 1, []namespace.ID{blobs[0].Namespace(), blobs[1].Namespace()}) + _, err = service.GetAll(ctx, 1, []share.Namespace{blobs[0].Namespace(), blobs[1].Namespace()}) require.NoError(t, err) } @@ -417,7 +401,7 @@ func createService(ctx context.Context, t *testing.T, blobs []*Blob) *Service { require.NoError(t, err) rawShares, err := BlobsToShares(blobs...) require.NoError(t, err) - eds, err := share.AddShares(ctx, rawShares, bs) + eds, err := ipld.AddShares(ctx, rawShares, bs) require.NoError(t, err) h := headertest.ExtendedHeaderFromEDS(t, 1, eds) diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index 18851dc797..169ca4b8d1 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -16,9 +16,6 @@ import ( "github.com/spf13/cobra" - appns "github.com/celestiaorg/celestia-app/pkg/namespace" - "github.com/celestiaorg/nmt/namespace" - "github.com/celestiaorg/celestia-node/api/rpc/client" "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/share" @@ -112,18 +109,18 @@ func parseParams(method string, params []string) []interface{} { panic(fmt.Errorf("couldn't parse share root as json: %v", err)) } parsedParams[0] = root - // 2. NamespaceID - nID, err := parseV0NamespaceID(params[1]) + // 2. Namespace + namespace, err := parseV0Namespace(params[1]) if err != nil { - panic(fmt.Sprintf("Error parsing namespace ID: %v", err)) + panic(fmt.Sprintf("Error parsing namespace: %v", err)) } - parsedParams[1] = nID + parsedParams[1] = namespace case "Submit": - // 1. NamespaceID + // 1. Namespace var err error - nID, err := parseV0NamespaceID(params[0]) + namespace, err := parseV0Namespace(params[0]) if err != nil { - panic(fmt.Sprintf("Error parsing namespace ID: %v", err)) + panic(fmt.Sprintf("Error parsing namespace: %v", err)) } // 2. Blob data var blobData []byte @@ -146,7 +143,7 @@ func parseParams(method string, params []string) []interface{} { panic("Error decoding blob data: base64 string could not be decoded.") } } - parsedBlob, err := blob.NewBlob(0, nID, blobData) + parsedBlob, err := blob.NewBlobV0(namespace, blobData) if err != nil { panic(fmt.Sprintf("Error creating blob: %v", err)) } @@ -162,10 +159,10 @@ func parseParams(method string, params []string) []interface{} { panic("Error parsing gas limit: uint64 could not be parsed.") } parsedParams[1] = num - // 3. NamespaceID - nID, err := parseV0NamespaceID(params[2]) + // 3. Namespace + namespace, err := parseV0Namespace(params[2]) if err != nil { - panic(fmt.Sprintf("Error parsing namespace ID: %v", err)) + panic(fmt.Sprintf("Error parsing namespace: %v", err)) } // 4. Blob data var blobData []byte @@ -188,7 +185,7 @@ func parseParams(method string, params []string) []interface{} { panic("Error decoding blob: base64 string could not be decoded.") } } - parsedBlob, err := blob.NewBlob(0, nID, blobData) + parsedBlob, err := blob.NewBlobV0(namespace, blobData) if err != nil { panic(fmt.Sprintf("Error creating blob: %v", err)) } @@ -202,11 +199,11 @@ func parseParams(method string, params []string) []interface{} { } parsedParams[0] = num // 2. NamespaceID - nID, err := parseV0NamespaceID(params[1]) + namespace, err := parseV0Namespace(params[1]) if err != nil { - panic(fmt.Sprintf("Error parsing namespace ID: %v", err)) + panic(fmt.Sprintf("Error parsing namespace: %v", err)) } - parsedParams[1] = nID + parsedParams[1] = namespace // 3: Commitment commitment, err := base64.StdEncoding.DecodeString(params[2]) if err != nil { @@ -221,12 +218,12 @@ func parseParams(method string, params []string) []interface{} { panic("Error parsing gas limit: uint64 could not be parsed.") } parsedParams[0] = num - // 2. NamespaceID - nID, err := parseV0NamespaceID(params[1]) + // 2. Namespace + namespace, err := parseV0Namespace(params[1]) if err != nil { - panic(fmt.Sprintf("Error parsing namespace ID: %v", err)) + panic(fmt.Sprintf("Error parsing namespace: %v", err)) } - parsedParams[1] = []namespace.ID{nID} + parsedParams[1] = []share.Namespace{namespace} return parsedParams case "QueryDelegation", "QueryUnbonding", "BalanceForAddress": var err error @@ -419,23 +416,17 @@ func parseSignatureForHelpstring(methodSig reflect.StructField) string { return simplifiedSignature } -// parseV0NamespaceID parses a namespace ID from a base64 or hex string. The param +// parseV0Namespace parses a namespace from a base64 or hex string. The param // is expected to be the user-specified portion of a v0 namespace ID (i.e. the // last 10 bytes). -func parseV0NamespaceID(param string) (namespace.ID, error) { +func parseV0Namespace(param string) (share.Namespace, error) { userBytes, err := decodeToBytes(param) if err != nil { return nil, err } - if len(userBytes) > appns.NamespaceVersionZeroIDSize { - return nil, fmt.Errorf( - "namespace ID %v is too large to be a v0 namespace, want <= %v bytes", - userBytes, appns.NamespaceVersionZeroIDSize, - ) - } // if the namespace ID is <= 10 bytes, left pad it with 0s - return share.NewNamespaceV0(userBytes) + return share.NewBlobNamespaceV0(userBytes) } // decodeToBytes decodes a Base64 or hex input string into a byte slice. diff --git a/cmd/celestia/rpc_test.go b/cmd/celestia/rpc_test.go index 9380a8b01e..53087646a7 100644 --- a/cmd/celestia/rpc_test.go +++ b/cmd/celestia/rpc_test.go @@ -5,21 +5,21 @@ import ( "github.com/stretchr/testify/assert" - "github.com/celestiaorg/nmt/namespace" + "github.com/celestiaorg/celestia-node/share" ) func Test_parseNamespaceID(t *testing.T) { type testCase struct { name string param string - want namespace.ID + want share.Namespace wantErr bool } testCases := []testCase{ { param: "0x0c204d39600fddd3", name: "8 byte hex encoded namespace ID gets left padded", - want: namespace.ID{ + want: share.Namespace{ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x20, 0x4d, 0x39, 0x60, 0xf, 0xdd, 0xd3, }, @@ -28,7 +28,7 @@ func Test_parseNamespaceID(t *testing.T) { { name: "10 byte hex encoded namespace ID", param: "0x42690c204d39600fddd3", - want: namespace.ID{ + want: share.Namespace{ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x42, 0x69, 0xc, 0x20, 0x4d, 0x39, 0x60, 0xf, 0xdd, 0xd3, }, @@ -37,7 +37,7 @@ func Test_parseNamespaceID(t *testing.T) { { name: "29 byte hex encoded namespace ID", param: "0x0000000000000000000000000000000000000001010101010101010101", - want: namespace.ID{ + want: share.Namespace{ 0x0, // namespace version 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // v0 ID prefix 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // namespace ID @@ -47,13 +47,13 @@ func Test_parseNamespaceID(t *testing.T) { { name: "11 byte hex encoded namespace ID returns error", param: "0x42690c204d39600fddd3a3", - want: namespace.ID{}, + want: share.Namespace{}, wantErr: true, }, { name: "10 byte base64 encoded namespace ID", param: "QmkMIE05YA/d0w==", - want: namespace.ID{ + want: share.Namespace{ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x42, 0x69, 0xc, 0x20, 0x4d, 0x39, 0x60, 0xf, 0xdd, 0xd3, }, @@ -62,14 +62,14 @@ func Test_parseNamespaceID(t *testing.T) { { name: "not base64 or hex encoded namespace ID returns error", param: "5748493939429", - want: namespace.ID{}, + want: share.Namespace{}, wantErr: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - got, err := parseV0NamespaceID(tc.param) + got, err := parseV0Namespace(tc.param) if tc.wantErr { assert.Error(t, err) return @@ -77,6 +77,5 @@ func Test_parseNamespaceID(t *testing.T) { assert.NoError(t, err) assert.Equal(t, tc.want, got) }) - } } diff --git a/go.mod b/go.mod index 31420f8745..3d38242260 100644 --- a/go.mod +++ b/go.mod @@ -10,11 +10,11 @@ require ( github.com/BurntSushi/toml v1.3.0 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.0.0-rc4 + github.com/celestiaorg/celestia-app v1.0.0-rc5 github.com/celestiaorg/go-fraud v0.1.0 github.com/celestiaorg/go-header v0.2.7 github.com/celestiaorg/go-libp2p-messenger v0.2.0 - github.com/celestiaorg/nmt v0.16.0 + github.com/celestiaorg/nmt v0.17.0 github.com/celestiaorg/rsmt2d v0.9.0 github.com/cosmos/cosmos-sdk v0.46.13 github.com/cosmos/cosmos-sdk/api v0.1.0 @@ -44,7 +44,7 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-merkledag v0.10.0 github.com/ipld/go-car v0.6.0 - github.com/libp2p/go-libp2p v0.28.0 + github.com/libp2p/go-libp2p v0.28.1 github.com/libp2p/go-libp2p-kad-dht v0.21.1 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-libp2p-record v0.2.0 diff --git a/go.sum b/go.sum index 3bf1455309..d8cfbe3806 100644 --- a/go.sum +++ b/go.sum @@ -344,8 +344,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.0.0-rc4 h1:S48+XMsLnk0GkNHrUc5c+rlyyHLIl0ZJ/zh92jmSSTk= -github.com/celestiaorg/celestia-app v1.0.0-rc4/go.mod h1:i+jE3Hh8IQxlZnE5XSBlYF0PCNPvD3v5g8KmY095PEA= +github.com/celestiaorg/celestia-app v1.0.0-rc5 h1:PjKIp91CQgqyP4MLI3KeF8E4MPHtBPlNoRPvGIMWjHw= +github.com/celestiaorg/celestia-app v1.0.0-rc5/go.mod h1:i+jE3Hh8IQxlZnE5XSBlYF0PCNPvD3v5g8KmY095PEA= github.com/celestiaorg/celestia-core v1.22.0-tm-v0.34.28 h1:idHJK9i4WCkYOf5PXVWZbOs8pWkCiHHQGI4MZr0iMtQ= github.com/celestiaorg/celestia-core v1.22.0-tm-v0.34.28/go.mod h1:LOxHW9nA++/9U8TgvTyKo9TO3F09sWv8asKQs00m73U= github.com/celestiaorg/cosmos-sdk v1.15.0-sdk-v0.46.13 h1:vaQKgaOm0w58JAvOgn2iDohqjH7kvvRqVKiMcBDWifA= @@ -362,8 +362,8 @@ github.com/celestiaorg/go-verifcid v0.0.1-lazypatch h1:9TSe3w1cmJmbWlweCwCTIZkan github.com/celestiaorg/go-verifcid v0.0.1-lazypatch/go.mod h1:kXPYu0XqTNUKWA1h3M95UHjUqBzDwXVVt/RXZDjKJmQ= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= -github.com/celestiaorg/nmt v0.16.0 h1:4CX6d1Uwf1C+tGcAWskPve0HCDTnI4Ey8ffjiDwcGH0= -github.com/celestiaorg/nmt v0.16.0/go.mod h1:GfwIvQPhUakn1modWxJ+rv8dUjJzuXg5H+MLFM1o7nY= +github.com/celestiaorg/nmt v0.17.0 h1:/k8YLwJvuHgT/jQ435zXKaDX811+sYEMXL4B/vYdSLU= +github.com/celestiaorg/nmt v0.17.0/go.mod h1:ZndCeAR4l9lxm7W51ouoyTo1cxhtFgK+4DpEIkxRA3A= github.com/celestiaorg/quantum-gravity-bridge v1.3.0 h1:9zPIp7w1FWfkPnn16y3S4FpFLnQtS7rm81CUVcHEts0= github.com/celestiaorg/quantum-gravity-bridge v1.3.0/go.mod h1:6WOajINTDEUXpSj5UZzod16UZ96ZVB/rFNKyM+Mt1gI= github.com/celestiaorg/rsmt2d v0.9.0 h1:kon78I748ZqjNzI8OAqPN+2EImuZuanj/6gTh8brX3o= @@ -1213,8 +1213,8 @@ github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniV github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= github.com/libp2p/go-libp2p v0.19.0/go.mod h1:Ki9jJXLO2YqrTIFxofV7Twyd3INWPT97+r8hGt7XPjI= -github.com/libp2p/go-libp2p v0.28.0 h1:zO8cY98nJiPzZpFv5w5gqqb8aVzt4ukQ0nVOSaaKhJ8= -github.com/libp2p/go-libp2p v0.28.0/go.mod h1:s3Xabc9LSwOcnv9UD4nORnXKTsWkPMkIMB/JIGXVnzk= +github.com/libp2p/go-libp2p v0.28.1 h1:YurK+ZAI6cKfASLJBVFkpVBdl3wGhFi6fusOt725ii8= +github.com/libp2p/go-libp2p v0.28.1/go.mod h1:s3Xabc9LSwOcnv9UD4nORnXKTsWkPMkIMB/JIGXVnzk= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= @@ -1950,14 +1950,14 @@ github.com/tidwall/btree v1.5.0 h1:iV0yVY/frd7r6qGBXfEYs7DH0gTDgrKTrDjS7xt/IyQ= github.com/tidwall/btree v1.5.0/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= diff --git a/header/headertest/testing.go b/header/headertest/testing.go index 3e0da71d69..fbc71f92f9 100644 --- a/header/headertest/testing.go +++ b/header/headertest/testing.go @@ -26,7 +26,8 @@ import ( "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" - "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/ipld" ) var log = logging.Logger("headertest") @@ -349,17 +350,15 @@ func ExtendedHeaderFromEDS(t *testing.T, height uint64, eds *rsmt2d.ExtendedData func CreateFraudExtHeader( t *testing.T, eh *header.ExtendedHeader, - dag blockservice.BlockService, + serv blockservice.BlockService, ) (*header.ExtendedHeader, *rsmt2d.ExtendedDataSquare) { - extended := share.RandEDS(t, 2) - shares := share.ExtractEDS(extended) - copy(shares[0][share.NamespaceSize:], shares[1][share.NamespaceSize:]) - extended, err := share.ImportShares(context.Background(), shares, dag) + square := edstest.RandByzantineEDS(t, 16) + err := ipld.ImportEDS(context.Background(), square, serv) require.NoError(t, err) - dah := da.NewDataAvailabilityHeader(extended) + dah := da.NewDataAvailabilityHeader(square) eh.DAH = &dah eh.RawHeader.DataHash = dah.Hash() - return eh, extended + return eh, square } type Subscriber struct { diff --git a/libs/utils/address.go b/libs/utils/address.go index a8170e44b9..ae52a03b16 100644 --- a/libs/utils/address.go +++ b/libs/utils/address.go @@ -29,15 +29,14 @@ func ValidateAddr(addr string) (string, error) { return addr, err } - if ip := net.ParseIP(addr); ip == nil { - addrs, err := net.LookupHost(addr) - if err != nil { - return addr, fmt.Errorf("could not resolve %v: %w", addr, err) - } - if len(addrs) == 0 { - return addr, fmt.Errorf("no IP addresses found for DNS record: %v", addr) - } - addr = addrs[0] + ip := net.ParseIP(addr) + if ip != nil { + return addr, nil } - return addr, nil + + resolved, err := net.ResolveIPAddr("ip4", addr) + if err != nil { + return addr, err + } + return resolved.String(), nil } diff --git a/nodebuilder/blob/blob.go b/nodebuilder/blob/blob.go index 7cbe312856..aae502824c 100644 --- a/nodebuilder/blob/blob.go +++ b/nodebuilder/blob/blob.go @@ -3,9 +3,8 @@ package blob import ( "context" - "github.com/celestiaorg/nmt/namespace" - "github.com/celestiaorg/celestia-node/blob" + "github.com/celestiaorg/celestia-node/share" ) var _ Module = (*API)(nil) @@ -19,23 +18,23 @@ type Module interface { // Uses default wallet registered on the Node. Submit(_ context.Context, _ []*blob.Blob) (height uint64, _ error) // Get retrieves the blob by commitment under the given namespace and height. - Get(_ context.Context, height uint64, _ namespace.ID, _ blob.Commitment) (*blob.Blob, error) + Get(_ context.Context, height uint64, _ share.Namespace, _ blob.Commitment) (*blob.Blob, error) // GetAll returns all blobs under the given namespaces and height. - GetAll(_ context.Context, height uint64, _ []namespace.ID) ([]*blob.Blob, error) + GetAll(_ context.Context, height uint64, _ []share.Namespace) ([]*blob.Blob, error) // GetProof retrieves proofs in the given namespaces at the given height by commitment. - GetProof(_ context.Context, height uint64, _ namespace.ID, _ blob.Commitment) (*blob.Proof, error) + GetProof(_ context.Context, height uint64, _ share.Namespace, _ blob.Commitment) (*blob.Proof, error) // Included checks whether a blob's given commitment(Merkle subtree root) is included at // given height and under the namespace. - Included(_ context.Context, height uint64, _ namespace.ID, _ *blob.Proof, _ blob.Commitment) (bool, error) + Included(_ context.Context, height uint64, _ share.Namespace, _ *blob.Proof, _ blob.Commitment) (bool, error) } type API struct { Internal struct { - Submit func(context.Context, []*blob.Blob) (uint64, error) `perm:"write"` - Get func(context.Context, uint64, namespace.ID, blob.Commitment) (*blob.Blob, error) `perm:"read"` - GetAll func(context.Context, uint64, []namespace.ID) ([]*blob.Blob, error) `perm:"read"` - GetProof func(context.Context, uint64, namespace.ID, blob.Commitment) (*blob.Proof, error) `perm:"read"` - Included func(context.Context, uint64, namespace.ID, *blob.Proof, blob.Commitment) (bool, error) `perm:"read"` + Submit func(context.Context, []*blob.Blob) (uint64, error) `perm:"write"` + Get func(context.Context, uint64, share.Namespace, blob.Commitment) (*blob.Blob, error) `perm:"read"` + GetAll func(context.Context, uint64, []share.Namespace) ([]*blob.Blob, error) `perm:"read"` + GetProof func(context.Context, uint64, share.Namespace, blob.Commitment) (*blob.Proof, error) `perm:"read"` + Included func(context.Context, uint64, share.Namespace, *blob.Proof, blob.Commitment) (bool, error) `perm:"read"` } } @@ -46,31 +45,31 @@ func (api *API) Submit(ctx context.Context, blobs []*blob.Blob) (uint64, error) func (api *API) Get( ctx context.Context, height uint64, - nID namespace.ID, + namespace share.Namespace, commitment blob.Commitment, ) (*blob.Blob, error) { - return api.Internal.Get(ctx, height, nID, commitment) + return api.Internal.Get(ctx, height, namespace, commitment) } -func (api *API) GetAll(ctx context.Context, height uint64, nIDs []namespace.ID) ([]*blob.Blob, error) { - return api.Internal.GetAll(ctx, height, nIDs) +func (api *API) GetAll(ctx context.Context, height uint64, namespaces []share.Namespace) ([]*blob.Blob, error) { + return api.Internal.GetAll(ctx, height, namespaces) } func (api *API) GetProof( ctx context.Context, height uint64, - nID namespace.ID, + namespace share.Namespace, commitment blob.Commitment, ) (*blob.Proof, error) { - return api.Internal.GetProof(ctx, height, nID, commitment) + return api.Internal.GetProof(ctx, height, namespace, commitment) } func (api *API) Included( ctx context.Context, height uint64, - nID namespace.ID, + namespace share.Namespace, proof *blob.Proof, commitment blob.Commitment, ) (bool, error) { - return api.Internal.Included(ctx, height, nID, proof, commitment) + return api.Internal.Included(ctx, height, namespace, proof, commitment) } diff --git a/nodebuilder/blob/mocks/api.go b/nodebuilder/blob/mocks/api.go index f99d1d8168..5cd34b74b6 100644 --- a/nodebuilder/blob/mocks/api.go +++ b/nodebuilder/blob/mocks/api.go @@ -8,9 +8,10 @@ import ( context "context" reflect "reflect" - blob "github.com/celestiaorg/celestia-node/blob" - namespace "github.com/celestiaorg/nmt/namespace" gomock "github.com/golang/mock/gomock" + + blob "github.com/celestiaorg/celestia-node/blob" + share "github.com/celestiaorg/celestia-node/share" ) // MockModule is a mock of Module interface. @@ -37,7 +38,7 @@ func (m *MockModule) EXPECT() *MockModuleMockRecorder { } // Get mocks base method. -func (m *MockModule) Get(arg0 context.Context, arg1 uint64, arg2 namespace.ID, arg3 blob.Commitment) (*blob.Blob, error) { +func (m *MockModule) Get(arg0 context.Context, arg1 uint64, arg2 share.Namespace, arg3 blob.Commitment) (*blob.Blob, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(*blob.Blob) @@ -52,7 +53,7 @@ func (mr *MockModuleMockRecorder) Get(arg0, arg1, arg2, arg3 interface{}) *gomoc } // GetAll mocks base method. -func (m *MockModule) GetAll(arg0 context.Context, arg1 uint64, arg2 []namespace.ID) ([]*blob.Blob, error) { +func (m *MockModule) GetAll(arg0 context.Context, arg1 uint64, arg2 []share.Namespace) ([]*blob.Blob, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAll", arg0, arg1, arg2) ret0, _ := ret[0].([]*blob.Blob) @@ -67,7 +68,7 @@ func (mr *MockModuleMockRecorder) GetAll(arg0, arg1, arg2 interface{}) *gomock.C } // GetProof mocks base method. -func (m *MockModule) GetProof(arg0 context.Context, arg1 uint64, arg2 namespace.ID, arg3 blob.Commitment) (*blob.Proof, error) { +func (m *MockModule) GetProof(arg0 context.Context, arg1 uint64, arg2 share.Namespace, arg3 blob.Commitment) (*blob.Proof, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetProof", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(*blob.Proof) @@ -82,7 +83,7 @@ func (mr *MockModuleMockRecorder) GetProof(arg0, arg1, arg2, arg3 interface{}) * } // Included mocks base method. -func (m *MockModule) Included(arg0 context.Context, arg1 uint64, arg2 namespace.ID, arg3 *blob.Proof, arg4 blob.Commitment) (bool, error) { +func (m *MockModule) Included(arg0 context.Context, arg1 uint64, arg2 share.Namespace, arg3 *blob.Proof, arg4 blob.Commitment) (bool, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Included", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(bool) diff --git a/nodebuilder/das/mocks/api.go b/nodebuilder/das/mocks/api.go index c4046e90e8..68ffaf3c8c 100644 --- a/nodebuilder/das/mocks/api.go +++ b/nodebuilder/das/mocks/api.go @@ -8,8 +8,9 @@ import ( context "context" reflect "reflect" - das "github.com/celestiaorg/celestia-node/das" gomock "github.com/golang/mock/gomock" + + das "github.com/celestiaorg/celestia-node/das" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/fraud/mocks/api.go b/nodebuilder/fraud/mocks/api.go index ba88131695..399f8746e1 100644 --- a/nodebuilder/fraud/mocks/api.go +++ b/nodebuilder/fraud/mocks/api.go @@ -8,9 +8,10 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + fraud "github.com/celestiaorg/celestia-node/nodebuilder/fraud" fraud0 "github.com/celestiaorg/go-fraud" - gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/header/mocks/api.go b/nodebuilder/header/mocks/api.go index 02529a8ef9..7d6661ff5d 100644 --- a/nodebuilder/header/mocks/api.go +++ b/nodebuilder/header/mocks/api.go @@ -8,10 +8,11 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + header "github.com/celestiaorg/celestia-node/header" header0 "github.com/celestiaorg/go-header" sync "github.com/celestiaorg/go-header/sync" - gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/node/mocks/api.go b/nodebuilder/node/mocks/api.go index d8789a771c..14357316dc 100644 --- a/nodebuilder/node/mocks/api.go +++ b/nodebuilder/node/mocks/api.go @@ -8,9 +8,10 @@ import ( context "context" reflect "reflect" - node "github.com/celestiaorg/celestia-node/nodebuilder/node" auth "github.com/filecoin-project/go-jsonrpc/auth" gomock "github.com/golang/mock/gomock" + + node "github.com/celestiaorg/celestia-node/nodebuilder/node" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/share/constructors.go b/nodebuilder/share/constructors.go index 1913b3d576..5cb0e41e53 100644 --- a/nodebuilder/share/constructors.go +++ b/nodebuilder/share/constructors.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/filecoin-project/dagstore" + "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/routing" @@ -18,6 +19,7 @@ import ( "github.com/celestiaorg/celestia-node/share/availability/light" "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/getters" + "github.com/celestiaorg/celestia-node/share/ipld" disc "github.com/celestiaorg/celestia-node/share/p2p/discovery" ) @@ -60,6 +62,15 @@ func ensureEmptyCARExists(ctx context.Context, store *eds.Store) error { return err } +// ensureEmptyEDSInBS checks if the given DAG contains an empty block data square. +// If it does not, it stores an empty block. This optimization exists to prevent +// redundant storing of empty block data so that it is only stored once and returned +// upon request for a block with an empty data square. +func ensureEmptyEDSInBS(ctx context.Context, bServ blockservice.BlockService) error { + _, err := ipld.AddShares(ctx, share.EmptyBlockShares(), bServ) + return err +} + func lightGetter( shrexGetter *getters.ShrexGetter, ipldGetter *getters.IPLDGetter, diff --git a/nodebuilder/share/mocks/api.go b/nodebuilder/share/mocks/api.go index 586c6dab4b..66baa23301 100644 --- a/nodebuilder/share/mocks/api.go +++ b/nodebuilder/share/mocks/api.go @@ -8,11 +8,11 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + da "github.com/celestiaorg/celestia-app/pkg/da" share "github.com/celestiaorg/celestia-node/share" - namespace "github.com/celestiaorg/nmt/namespace" rsmt2d "github.com/celestiaorg/rsmt2d" - gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. @@ -69,7 +69,7 @@ func (mr *MockModuleMockRecorder) GetShare(arg0, arg1, arg2, arg3 interface{}) * } // GetSharesByNamespace mocks base method. -func (m *MockModule) GetSharesByNamespace(arg0 context.Context, arg1 *da.DataAvailabilityHeader, arg2 namespace.ID) (share.NamespacedShares, error) { +func (m *MockModule) GetSharesByNamespace(arg0 context.Context, arg1 *da.DataAvailabilityHeader, arg2 share.Namespace) (share.NamespacedShares, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSharesByNamespace", arg0, arg1, arg2) ret0, _ := ret[0].(share.NamespacedShares) diff --git a/nodebuilder/share/module.go b/nodebuilder/share/module.go index 6dfb155bb0..b924cf8167 100644 --- a/nodebuilder/share/module.go +++ b/nodebuilder/share/module.go @@ -176,7 +176,7 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option } }), shrexGetterComponents, - fx.Invoke(share.EnsureEmptySquareExists), + fx.Invoke(ensureEmptyEDSInBS), fx.Provide(getters.NewIPLDGetter), fx.Provide(lightGetter), // shrexsub broadcaster stub for daser diff --git a/nodebuilder/share/share.go b/nodebuilder/share/share.go index 0c703f9a15..def3bb4d0f 100644 --- a/nodebuilder/share/share.go +++ b/nodebuilder/share/share.go @@ -3,7 +3,6 @@ package share import ( "context" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" @@ -40,7 +39,7 @@ type Module interface { GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.ExtendedDataSquare, error) // GetSharesByNamespace gets all shares from an EDS within the given namespace. // Shares are returned in a row-by-row order if the namespace spans multiple rows. - GetSharesByNamespace(ctx context.Context, root *share.Root, namespace namespace.ID) (share.NamespacedShares, error) + GetSharesByNamespace(ctx context.Context, root *share.Root, namespace share.Namespace) (share.NamespacedShares, error) } // API is a wrapper around Module for the RPC. @@ -61,7 +60,7 @@ type API struct { GetSharesByNamespace func( ctx context.Context, root *share.Root, - namespace namespace.ID, + namespace share.Namespace, ) (share.NamespacedShares, error) `perm:"public"` } } @@ -85,7 +84,7 @@ func (api *API) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.ExtendedD func (api *API) GetSharesByNamespace( ctx context.Context, root *share.Root, - namespace namespace.ID, + namespace share.Namespace, ) (share.NamespacedShares, error) { return api.Internal.GetSharesByNamespace(ctx, root, namespace) } diff --git a/nodebuilder/state/mocks/api.go b/nodebuilder/state/mocks/api.go index dbd1d5dabe..754920dee2 100644 --- a/nodebuilder/state/mocks/api.go +++ b/nodebuilder/state/mocks/api.go @@ -9,12 +9,13 @@ import ( reflect "reflect" math "cosmossdk.io/math" - blob "github.com/celestiaorg/celestia-node/blob" - state "github.com/celestiaorg/celestia-node/state" types "github.com/cosmos/cosmos-sdk/types" types0 "github.com/cosmos/cosmos-sdk/x/staking/types" gomock "github.com/golang/mock/gomock" types1 "github.com/tendermint/tendermint/types" + + blob "github.com/celestiaorg/celestia-node/blob" + state "github.com/celestiaorg/celestia-node/state" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/tests/api_test.go b/nodebuilder/tests/api_test.go index 37504ec3b3..1bc1c261de 100644 --- a/nodebuilder/tests/api_test.go +++ b/nodebuilder/tests/api_test.go @@ -109,7 +109,7 @@ func TestBlobRPC(t *testing.T) { client, err := client.NewClient(ctx, bridgeAddr, jwt) require.NoError(t, err) - appBlobs, err := blobtest.GenerateBlobs([]int{8}, false) + appBlobs, err := blobtest.GenerateV0Blobs([]int{8}, false) require.NoError(t, err) newBlob, err := blob.NewBlob( diff --git a/nodebuilder/tests/blob_test.go b/nodebuilder/tests/blob_test.go index 537e320c5a..6a7db50c7c 100644 --- a/nodebuilder/tests/blob_test.go +++ b/nodebuilder/tests/blob_test.go @@ -11,12 +11,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/celestiaorg/nmt/namespace" - "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/blob/blobtest" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" + "github.com/celestiaorg/celestia-node/share" ) func TestBlobModule(t *testing.T) { @@ -24,9 +23,9 @@ func TestBlobModule(t *testing.T) { t.Cleanup(cancel) sw := swamp.NewSwamp(t) - appBlobs0, err := blobtest.GenerateBlobs([]int{8, 4}, true) + appBlobs0, err := blobtest.GenerateV0Blobs([]int{8, 4}, true) require.NoError(t, err) - appBlobs1, err := blobtest.GenerateBlobs([]int{4}, false) + appBlobs1, err := blobtest.GenerateV0Blobs([]int{4}, false) require.NoError(t, err) blobs := make([]*blob.Blob, 0, len(appBlobs0)+len(appBlobs1)) @@ -79,7 +78,7 @@ func TestBlobModule(t *testing.T) { { name: "GetAll", doFn: func(t *testing.T) { - newBlobs, err := fullNode.BlobServ.GetAll(ctx, height, []namespace.ID{blobs[0].Namespace()}) + newBlobs, err := fullNode.BlobServ.GetAll(ctx, height, []share.Namespace{blobs[0].Namespace()}) require.NoError(t, err) require.Len(t, newBlobs, len(appBlobs0)) require.True(t, bytes.Equal(blobs[0].Commitment, newBlobs[0].Commitment)) @@ -106,7 +105,7 @@ func TestBlobModule(t *testing.T) { { name: "Not Found", doFn: func(t *testing.T) { - appBlob, err := blobtest.GenerateBlobs([]int{4}, false) + appBlob, err := blobtest.GenerateV0Blobs([]int{4}, false) require.NoError(t, err) newBlob, err := blob.NewBlob( appBlob[0].ShareVersion, diff --git a/share/availability.go b/share/availability.go index 9538573114..b6b44271c8 100644 --- a/share/availability.go +++ b/share/availability.go @@ -4,7 +4,7 @@ import ( "context" "errors" - da "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/celestia-app/pkg/da" ) // ErrNotAvailable is returned whenever DA sampling fails. diff --git a/share/availability/light/availability_test.go b/share/availability/light/availability_test.go index ca72596816..1709c3f4b7 100644 --- a/share/availability/light/availability_test.go +++ b/share/availability/light/availability_test.go @@ -10,11 +10,11 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" + "github.com/celestiaorg/celestia-node/share/sharetest" ) func TestSharesAvailable(t *testing.T) { @@ -81,23 +81,23 @@ func TestService_GetSharesByNamespace(t *testing.T) { t.Run("size: "+strconv.Itoa(tt.squareSize), func(t *testing.T) { getter, bServ := EmptyGetter() totalShares := tt.squareSize * tt.squareSize - randShares := share.RandShares(t, totalShares) + randShares := sharetest.RandShares(t, totalShares) idx1 := (totalShares - 1) / 2 idx2 := totalShares / 2 if tt.expectedShareCount > 1 { - // make it so that two rows have the same namespace ID - copy(randShares[idx2][:namespace.NamespaceSize], randShares[idx1][:namespace.NamespaceSize]) + // make it so that two rows have the same namespace + copy(share.GetNamespace(randShares[idx2]), share.GetNamespace(randShares[idx1])) } root := availability_test.FillBS(t, bServ, randShares) - randNID := randShares[idx1][:namespace.NamespaceSize] + randNamespace := share.GetNamespace(randShares[idx1]) - shares, err := getter.GetSharesByNamespace(context.Background(), root, randNID) + shares, err := getter.GetSharesByNamespace(context.Background(), root, randNamespace) require.NoError(t, err) - require.NoError(t, shares.Verify(root, randNID)) + require.NoError(t, shares.Verify(root, randNamespace)) flattened := shares.Flatten() assert.Len(t, flattened, tt.expectedShareCount) for _, value := range flattened { - assert.Equal(t, randNID, []byte(share.ID(value))) + assert.Equal(t, randNamespace, share.GetNamespace(value)) } if tt.expectedShareCount > 1 { // idx1 is always smaller than idx2 @@ -105,14 +105,14 @@ func TestService_GetSharesByNamespace(t *testing.T) { assert.Equal(t, randShares[idx2], flattened[1]) } }) - t.Run("last two rows of a 4x4 square that have the same namespace ID have valid NMT proofs", func(t *testing.T) { + t.Run("last two rows of a 4x4 square that have the same namespace have valid NMT proofs", func(t *testing.T) { squareSize := 4 totalShares := squareSize * squareSize getter, bServ := EmptyGetter() - randShares := share.RandShares(t, totalShares) - lastNID := randShares[totalShares-1][:namespace.NamespaceSize] + randShares := sharetest.RandShares(t, totalShares) + lastNID := share.GetNamespace(randShares[totalShares-1]) for i := totalShares / 2; i < totalShares; i++ { - copy(randShares[i][:namespace.NamespaceSize], lastNID) + copy(share.GetNamespace(randShares[i]), lastNID) } root := availability_test.FillBS(t, bServ, randShares) @@ -141,7 +141,7 @@ func TestService_GetSharesByNamespaceNotFound(t *testing.T) { getter, root := GetterWithRandSquare(t, 1) root.RowRoots = nil - emptyShares, err := getter.GetSharesByNamespace(context.Background(), root, namespace.RandomNamespace().Bytes()) + emptyShares, err := getter.GetSharesByNamespace(context.Background(), root, sharetest.RandV0Namespace()) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) } @@ -159,11 +159,11 @@ func BenchmarkService_GetSharesByNamespace(b *testing.B) { b.Run(strconv.Itoa(tt.amountShares), func(b *testing.B) { t := &testing.T{} getter, root := GetterWithRandSquare(t, tt.amountShares) - randNID := root.RowRoots[(len(root.RowRoots)-1)/2][:8] + randNamespace := root.RowRoots[(len(root.RowRoots)-1)/2][:share.NamespaceSize] root.RowRoots[(len(root.RowRoots) / 2)] = root.RowRoots[(len(root.RowRoots)-1)/2] b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := getter.GetSharesByNamespace(context.Background(), root, randNID) + _, err := getter.GetSharesByNamespace(context.Background(), root, randNamespace) require.NoError(t, err) } }) diff --git a/share/availability/test/testing.go b/share/availability/test/testing.go index b2fb780bd4..19f63f114a 100644 --- a/share/availability/test/testing.go +++ b/share/availability/test/testing.go @@ -20,17 +20,19 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/sharetest" ) // RandFillBS fills the given BlockService with a random block of a given size. func RandFillBS(t *testing.T, n int, bServ blockservice.BlockService) *share.Root { - shares := share.RandShares(t, n*n) + shares := sharetest.RandShares(t, n*n) return FillBS(t, bServ, shares) } // FillBS fills the given BlockService with the given shares. func FillBS(t *testing.T, bServ blockservice.BlockService, shares []share.Share) *share.Root { - eds, err := share.AddShares(context.TODO(), shares, bServ) + eds, err := ipld.AddShares(context.TODO(), shares, bServ) require.NoError(t, err) dah := da.NewDataAvailabilityHeader(eds) return &dah diff --git a/share/doc.go b/share/doc.go index 7ef8c9f7bf..97229932a7 100644 --- a/share/doc.go +++ b/share/doc.go @@ -4,7 +4,7 @@ block data. Though this package contains several useful methods for getting specific shares and/or sampling them at random, a particularly useful method is GetSharesByNamespace which retrieves -all shares of block data of the given namespace.ID from the block associated with the given +all shares of block data of the given Namespace from the block associated with the given DataAvailabilityHeader (DAH, but referred to as Root within this package). This package also contains declaration of the Availability interface. Implementations of diff --git a/share/eds/byzantine/bad_encoding.go b/share/eds/byzantine/bad_encoding.go index 4e673ed8b8..0672096b25 100644 --- a/share/eds/byzantine/bad_encoding.go +++ b/share/eds/byzantine/bad_encoding.go @@ -143,17 +143,17 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error { // verify that Merkle proofs correspond to particular shares. shares := make([][]byte, len(merkleRowRoots)) - for index, share := range p.Shares { - if share == nil { + for index, shr := range p.Shares { + if shr == nil { continue } // validate inclusion of the share into one of the DAHeader roots - if ok := share.Validate(ipld.MustCidFromNamespacedSha256(root)); !ok { + if ok := shr.Validate(ipld.MustCidFromNamespacedSha256(root)); !ok { return fmt.Errorf("fraud: invalid proof: incorrect share received at index %d", index) } // NMTree commits the additional namespace while rsmt2d does not know about, so we trim it // this is ugliness from NMTWrapper that we have to embrace ¯\_(ツ)_/¯ - shares[index] = share.Share[ipld.NamespaceSize:] + shares[index] = share.GetData(shr.Share) } odsWidth := uint64(len(merkleRowRoots) / 2) diff --git a/share/eds/byzantine/bad_encoding_test.go b/share/eds/byzantine/bad_encoding_test.go index 6e413b17a2..5f6adac595 100644 --- a/share/eds/byzantine/bad_encoding_test.go +++ b/share/eds/byzantine/bad_encoding_test.go @@ -3,8 +3,10 @@ package byzantine import ( "context" "testing" + "time" mdutils "github.com/ipfs/go-merkledag/test" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" core "github.com/tendermint/tendermint/types" @@ -12,10 +14,34 @@ import ( "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" - "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/sharetest" ) +func TestBadEncodingFraudProof(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + defer t.Cleanup(cancel) + bServ := mdutils.Bserv() + + square := edstest.RandByzantineEDS(t, 16) + dah := da.NewDataAvailabilityHeader(square) + err := ipld.ImportEDS(ctx, square, bServ) + require.NoError(t, err) + + var errRsmt2d *rsmt2d.ErrByzantineData + err = square.Repair(dah.RowRoots, dah.ColumnRoots) + require.ErrorAs(t, err, &errRsmt2d) + + errByz := NewErrByzantine(ctx, bServ, &dah, errRsmt2d) + + befp := CreateBadEncodingProof([]byte("hash"), 0, errByz) + err = befp.Validate(&header.ExtendedHeader{ + DAH: &dah, + }) + assert.NoError(t, err) +} + // TestIncorrectBadEncodingFraudProof asserts that BEFP is not generated for the correct data func TestIncorrectBadEncodingFraudProof(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) @@ -24,9 +50,9 @@ func TestIncorrectBadEncodingFraudProof(t *testing.T) { bServ := mdutils.Bserv() squareSize := 8 - shares := share.RandShares(t, squareSize*squareSize) + shares := sharetest.RandShares(t, squareSize*squareSize) - eds, err := share.AddShares(ctx, shares, bServ) + eds, err := ipld.AddShares(ctx, shares, bServ) require.NoError(t, err) dah := da.NewDataAvailabilityHeader(eds) diff --git a/share/eds/byzantine/share_proof.go b/share/eds/byzantine/share_proof.go index 86eaad54a8..b8e39ee1d3 100644 --- a/share/eds/byzantine/share_proof.go +++ b/share/eds/byzantine/share_proof.go @@ -45,8 +45,8 @@ func NewShareWithProof(index int, share share.Share, pathToLeaf []cid.Cid) *Shar func (s *ShareWithProof) Validate(root cid.Cid) bool { return s.Proof.VerifyInclusion( sha256.New(), // TODO(@Wondertan): This should be defined somewhere globally - share.ID(s.Share), - [][]byte{share.Data(s.Share)}, + share.GetNamespace(s.Share).ToNMT(), + [][]byte{share.GetData(s.Share)}, ipld.NamespacedSha256FromCID(root), ) } diff --git a/share/eds/byzantine/share_proof_test.go b/share/eds/byzantine/share_proof_test.go index 9cffe6eb18..0f63d4f0c9 100644 --- a/share/eds/byzantine/share_proof_test.go +++ b/share/eds/byzantine/share_proof_test.go @@ -12,8 +12,8 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/sharetest" ) func TestGetProof(t *testing.T) { @@ -23,8 +23,8 @@ func TestGetProof(t *testing.T) { defer cancel() bServ := mdutils.Bserv() - shares := share.RandShares(t, width*width) - in, err := share.AddShares(ctx, shares, bServ) + shares := sharetest.RandShares(t, width*width) + in, err := ipld.AddShares(ctx, shares, bServ) require.NoError(t, err) dah := da.NewDataAvailabilityHeader(in) @@ -59,8 +59,8 @@ func TestGetProofs(t *testing.T) { defer cancel() bServ := mdutils.Bserv() - shares := share.RandShares(t, width*width) - in, err := share.AddShares(ctx, shares, bServ) + shares := sharetest.RandShares(t, width*width) + in, err := ipld.AddShares(ctx, shares, bServ) require.NoError(t, err) dah := da.NewDataAvailabilityHeader(in) diff --git a/share/eds/eds.go b/share/eds/eds.go index 4e96fd684e..544f3c2438 100644 --- a/share/eds/eds.go +++ b/share/eds/eds.go @@ -18,9 +18,7 @@ import ( "github.com/ipld/go-car/util" "github.com/minio/sha256-simd" - "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" @@ -37,7 +35,7 @@ var ErrEmptySquare = errors.New("share: importing empty data") type writingSession struct { eds *rsmt2d.ExtendedDataSquare store bstore.Blockstore // caches inner nodes (proofs) while we walk the nmt tree. - hasher *nmt.Hasher + hasher *nmt.NmtHasher w io.Writer } @@ -114,7 +112,7 @@ func initializeWriter(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io. return &writingSession{ eds: eds, store: store, - hasher: nmt.NewNmtHasher(sha256.New(), ipld.NamespaceSize, ipld.NMTIgnoreMaxNamespace), + hasher: nmt.NewNmtHasher(sha256.New(), share.NamespaceSize, ipld.NMTIgnoreMaxNamespace), w: w, }, nil } @@ -219,13 +217,13 @@ func getQuadrantCells(eds *rsmt2d.ExtendedDataSquare, i, j uint) [][]byte { // prependNamespace adds the namespace to the passed share if in the first quadrant, // otherwise it adds the ParitySharesNamespace to the beginning. -func prependNamespace(quadrant int, share []byte) []byte { - namespacedShare := make([]byte, 0, appconsts.NamespaceSize+appconsts.ShareSize) +func prependNamespace(quadrant int, shr share.Share) []byte { + namespacedShare := make([]byte, 0, share.NamespaceSize+share.Size) switch quadrant { case 0: - return append(append(namespacedShare, share[:ipld.NamespaceSize]...), share...) + return append(append(namespacedShare, share.GetNamespace(shr)...), shr...) case 1, 2, 3: - return append(append(namespacedShare, namespace.ParitySharesNamespace.Bytes()...), share...) + return append(append(namespacedShare, share.ParitySharesNamespace...), shr...) default: panic("invalid quadrant") } @@ -273,7 +271,7 @@ func ReadEDS(ctx context.Context, r io.Reader, root share.DataHash) (eds *rsmt2d } // the stored first quadrant shares are wrapped with the namespace twice. // we cut it off here, because it is added again while importing to the tree below - shares[i] = block.RawData()[ipld.NamespaceSize:] + shares[i] = share.GetData(block.RawData()) } eds, err = rsmt2d.ComputeExtendedDataSquare( diff --git a/share/eds/eds_test.go b/share/eds/eds_test.go index ea0f06c138..0ef211ec6f 100644 --- a/share/eds/eds_test.go +++ b/share/eds/eds_test.go @@ -18,10 +18,10 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/ipld" ) @@ -57,7 +57,7 @@ func TestQuadrantOrder(t *testing.T) { res := quadrantOrder(eds) for _, s := range res { - require.Len(t, s, testShareSize+namespace.NamespaceSize) + require.Len(t, s, testShareSize+share.NamespaceSize) } for q := 0; q < 4; q++ { @@ -101,7 +101,7 @@ func TestWriteEDSStartsWithLeaves(t *testing.T) { block, err := reader.Next() require.NoError(t, err, "error getting first block") - require.Equal(t, block.RawData()[ipld.NamespaceSize:], eds.GetCell(0, 0)) + require.Equal(t, share.GetData(block.RawData()), eds.GetCell(0, 0)) } func TestWriteEDSIncludesRoots(t *testing.T) { @@ -193,7 +193,7 @@ func TestReadEDS(t *testing.T) { func TestReadEDSContentIntegrityMismatch(t *testing.T) { writeRandomEDS(t) - dah := da.NewDataAvailabilityHeader(share.RandEDS(t, 4)) + dah := da.NewDataAvailabilityHeader(edstest.RandEDS(t, 4)) f := openWrittenEDS(t) defer f.Close() @@ -208,7 +208,7 @@ func BenchmarkReadWriteEDS(b *testing.B) { ctx, cancel := context.WithCancel(context.Background()) b.Cleanup(cancel) for originalDataWidth := 4; originalDataWidth <= 64; originalDataWidth *= 2 { - eds := share.RandEDS(b, originalDataWidth) + eds := edstest.RandEDS(b, originalDataWidth) dah := da.NewDataAvailabilityHeader(eds) b.Run(fmt.Sprintf("Writing %dx%d", originalDataWidth, originalDataWidth), func(b *testing.B) { b.ReportAllocs() @@ -242,7 +242,7 @@ func writeRandomEDS(t *testing.T) *rsmt2d.ExtendedDataSquare { f, err := os.OpenFile("test.car", os.O_WRONLY|os.O_CREATE, 0600) require.NoError(t, err, "error opening file") - eds := share.RandEDS(t, 4) + eds := edstest.RandEDS(t, 4) err = WriteEDS(ctx, eds, f) require.NoError(t, err, "error writing EDS to file") f.Close() @@ -276,7 +276,7 @@ func createTestData(t *testing.T, testDir string) { //nolint:unused f, err := os.OpenFile("example.car", os.O_WRONLY|os.O_CREATE, 0600) require.NoError(t, err, "opening file") - eds := share.RandEDS(t, 4) + eds := edstest.RandEDS(t, 4) err = WriteEDS(ctx, eds, f) require.NoError(t, err, "writing EDS to file") f.Close() diff --git a/share/eds/edstest/testing.go b/share/eds/edstest/testing.go new file mode 100644 index 0000000000..2b6d9ef78d --- /dev/null +++ b/share/eds/edstest/testing.go @@ -0,0 +1,31 @@ +package edstest + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/sharetest" +) + +func RandByzantineEDS(t *testing.T, size int) *rsmt2d.ExtendedDataSquare { + eds := RandEDS(t, size) + shares := share.ExtractEDS(eds) + copy(share.GetData(shares[0]), share.GetData(shares[1])) // corrupting eds + eds, err := rsmt2d.ImportExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(size))) + require.NoError(t, err, "failure to recompute the extended data square") + return eds +} + +// RandEDS generates EDS filled with the random data with the given size for original square. It +// uses require.TestingT to be able to take both a *testing.T and a *testing.B. +func RandEDS(t require.TestingT, size int) *rsmt2d.ExtendedDataSquare { + shares := sharetest.RandShares(t, size*size) + eds, err := rsmt2d.ComputeExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(size))) + require.NoError(t, err, "failure to recompute the extended data square") + return eds +} diff --git a/share/eds/ods_test.go b/share/eds/ods_test.go index 75fad3022e..eb243a4022 100644 --- a/share/eds/ods_test.go +++ b/share/eds/ods_test.go @@ -52,7 +52,7 @@ func TestODSReader(t *testing.T) { assert.NoError(t, err) // check that original data from eds is same as data from reader - assert.Equal(t, original, block.RawData()[share.NamespaceSize:]) + assert.Equal(t, original, share.GetData(block.RawData())) } } diff --git a/share/eds/retriever.go b/share/eds/retriever.go index b2dcc4ff7a..b3cf363056 100644 --- a/share/eds/retriever.go +++ b/share/eds/retriever.go @@ -256,7 +256,7 @@ func (rs *retrievalSession) doRequest(ctx context.Context, q *quadrant) { // and go get shares of left or the right side of the whole col/row axis // the left or the right side of the tree represent some portion of the quadrant // which we put into the rs.square share-by-share by calculating shares' indexes using q.index - share.GetShares(ctx, rs.bget, nd.Links()[q.x].Cid, size, func(j int, share share.Share) { + ipld.GetShares(ctx, rs.bget, nd.Links()[q.x].Cid, size, func(j int, share share.Share) { // NOTE: Each share can appear twice here, for a Row and Col, respectively. // These shares are always equal, and we allow only the first one to be written // in the square. diff --git a/share/eds/retriever_test.go b/share/eds/retriever_test.go index e90216de13..ca689aa5ec 100644 --- a/share/eds/retriever_test.go +++ b/share/eds/retriever_test.go @@ -2,11 +2,9 @@ package eds import ( "context" - "errors" "testing" "time" - "github.com/ipfs/go-blockservice" mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -16,11 +14,11 @@ import ( "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" - "github.com/celestiaorg/celestia-node/header" - "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds/byzantine" + "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/sharetest" ) func TestRetriever_Retrieve(t *testing.T) { @@ -48,8 +46,8 @@ func TestRetriever_Retrieve(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { // generate EDS - shares := share.RandShares(t, tc.squareSize*tc.squareSize) - in, err := share.AddShares(ctx, shares, bServ) + shares := sharetest.RandShares(t, tc.squareSize*tc.squareSize) + in, err := ipld.AddShares(ctx, shares, bServ) require.NoError(t, err) // limit with timeout, specifically retrieval @@ -70,8 +68,8 @@ func TestRetriever_ByzantineError(t *testing.T) { defer cancel() bserv := mdutils.Bserv() - shares := share.ExtractEDS(share.RandEDS(t, width)) - _, err := share.ImportShares(ctx, shares, bserv) + shares := share.ExtractEDS(edstest.RandEDS(t, width)) + _, err := ipld.ImportShares(ctx, shares, bserv) require.NoError(t, err) // corrupt shares so that eds erasure coding does not match @@ -109,8 +107,8 @@ func TestRetriever_MultipleRandQuadrants(t *testing.T) { r := NewRetriever(bServ) // generate EDS - shares := share.RandShares(t, squareSize*squareSize) - in, err := share.AddShares(ctx, shares, bServ) + shares := sharetest.RandShares(t, squareSize*squareSize) + in, err := ipld.AddShares(ctx, shares, bServ) require.NoError(t, err) dah := da.NewDataAvailabilityHeader(in) @@ -126,31 +124,3 @@ func TestRetriever_MultipleRandQuadrants(t *testing.T) { _, err = ses.Reconstruct(ctx) assert.NoError(t, err) } - -func TestFraudProofValidation(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) - defer t.Cleanup(cancel) - bServ := mdutils.Bserv() - - var errByz *byzantine.ErrByzantine - faultHeader, err := generateByzantineError(ctx, t, bServ) - require.True(t, errors.As(err, &errByz)) - - p := byzantine.CreateBadEncodingProof([]byte("hash"), uint64(faultHeader.Height()), errByz) - err = p.Validate(faultHeader) - require.NoError(t, err) -} - -func generateByzantineError( - ctx context.Context, - t *testing.T, - bServ blockservice.BlockService, -) (*header.ExtendedHeader, error) { - store := headertest.NewStore(t) - h, err := store.GetByHeight(ctx, 1) - require.NoError(t, err) - - faultHeader, _ := headertest.CreateFraudExtHeader(t, h, bServ) - _, err = NewRetriever(bServ).Retrieve(ctx, faultHeader.DAH) - return faultHeader, err -} diff --git a/share/eds/store_test.go b/share/eds/store_test.go index 4eb747d43e..536753eaa6 100644 --- a/share/eds/store_test.go +++ b/share/eds/store_test.go @@ -17,6 +17,7 @@ import ( "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/edstest" ) func TestEDSStore(t *testing.T) { @@ -77,7 +78,7 @@ func TestEDSStore(t *testing.T) { original := eds.GetCell(uint(i), uint(j)) block, err := carReader.Next() assert.NoError(t, err) - assert.Equal(t, original, block.RawData()[share.NamespaceSize:]) + assert.Equal(t, original, share.GetData(block.RawData())) } } }) @@ -304,7 +305,7 @@ func newStore(t *testing.T) (*Store, error) { } func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, share.Root) { - eds := share.RandEDS(t, 4) + eds := edstest.RandEDS(t, 4) dah := da.NewDataAvailabilityHeader(eds) return eds, dah diff --git a/share/empty.go b/share/empty.go index 0b7ea2e775..b0e1e6e6ae 100644 --- a/share/empty.go +++ b/share/empty.go @@ -2,10 +2,8 @@ package share import ( "bytes" - "context" "fmt" - - "github.com/ipfs/go-blockservice" + "sync" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" @@ -13,19 +11,48 @@ import ( "github.com/celestiaorg/rsmt2d" ) +// EmptyRoot returns Root of the empty block EDS. +func EmptyRoot() *Root { + initEmpty() + return emptyBlockRoot +} + +// EmptyExtendedDataSquare returns the EDS of the empty block data square. +func EmptyExtendedDataSquare() *rsmt2d.ExtendedDataSquare { + initEmpty() + return emptyBlockEDS +} + +// EmptyBlockShares returns the shares of the empty block. +func EmptyBlockShares() []Share { + initEmpty() + return emptyBlockShares +} + var ( - emptyRoot *Root - emptyEDS *rsmt2d.ExtendedDataSquare + emptyMu sync.Mutex + emptyBlockRoot *Root + emptyBlockEDS *rsmt2d.ExtendedDataSquare + emptyBlockShares []Share ) -func init() { +// initEmpty enables lazy initialization for constant empty block data. +func initEmpty() { + emptyMu.Lock() + defer emptyMu.Unlock() + if emptyBlockRoot != nil { + return + } + // compute empty block EDS and DAH for it - shares := emptyDataSquare() - eds, err := da.ExtendShares(shares) + result := shares.TailPaddingShares(appconsts.MinShareCount) + emptyBlockShares = shares.ToBytes(result) + + eds, err := da.ExtendShares(emptyBlockShares) if err != nil { panic(fmt.Errorf("failed to create empty EDS: %w", err)) } - emptyEDS = eds + emptyBlockEDS = eds dah := da.NewDataAvailabilityHeader(eds) minDAH := da.MinDataAvailabilityHeader() @@ -33,33 +60,8 @@ func init() { panic(fmt.Sprintf("mismatch in calculated minimum DAH and minimum DAH from celestia-app, "+ "expected %s, got %s", minDAH.String(), dah.String())) } - emptyRoot = &dah + emptyBlockRoot = &dah // precompute Hash, so it's cached internally to avoid potential races - emptyRoot.Hash() -} - -// EmptyRoot returns Root of an empty EDS. -func EmptyRoot() *Root { - return emptyRoot -} - -// EnsureEmptySquareExists checks if the given DAG contains an empty block data square. -// If it does not, it stores an empty block. This optimization exists to prevent -// redundant storing of empty block data so that it is only stored once and returned -// upon request for a block with an empty data square. Ref: header/constructors.go#L56 -func EnsureEmptySquareExists(ctx context.Context, bServ blockservice.BlockService) (*rsmt2d.ExtendedDataSquare, error) { - shares := emptyDataSquare() - return AddShares(ctx, shares, bServ) -} - -// EmptyExtendedDataSquare returns the EDS of the empty block data square. -func EmptyExtendedDataSquare() *rsmt2d.ExtendedDataSquare { - return emptyEDS -} - -// emptyDataSquare returns the minimum size data square filled with tail padding. -func emptyDataSquare() [][]byte { - result := shares.TailPaddingShares(appconsts.MinShareCount) - return shares.ToBytes(result) + emptyBlockRoot.Hash() } diff --git a/share/getter.go b/share/getter.go index 15f384a2c4..7caa954a24 100644 --- a/share/getter.go +++ b/share/getter.go @@ -8,10 +8,7 @@ import ( "github.com/minio/sha256-simd" "github.com/celestiaorg/nmt" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" - - "github.com/celestiaorg/celestia-node/share/ipld" ) var ( @@ -35,7 +32,7 @@ type Getter interface { // Inclusion of returned data could be verified using Verify method on NamespacedShares. // If no shares are found for target namespace non-inclusion could be also verified by calling // Verify method. - GetSharesByNamespace(context.Context, *Root, namespace.ID) (NamespacedShares, error) + GetSharesByNamespace(context.Context, *Root, Namespace) (NamespacedShares, error) } // NamespacedShares represents all shares with proofs within a specific namespace of an EDS. @@ -57,10 +54,10 @@ type NamespacedRow struct { } // Verify validates NamespacedShares by checking every row with nmt inclusion proof. -func (ns NamespacedShares) Verify(root *Root, nID namespace.ID) error { - originalRoots := make([][]byte, 0) +func (ns NamespacedShares) Verify(root *Root, namespace Namespace) error { + var originalRoots [][]byte for _, row := range root.RowRoots { - if !ipld.NamespaceIsOutsideRange(row, row, nID) { + if !namespace.IsOutsideRange(row, row) { originalRoots = append(originalRoots, row) } } @@ -72,7 +69,7 @@ func (ns NamespacedShares) Verify(root *Root, nID namespace.ID) error { for i, row := range ns { // verify row data against row hash from original root - if !row.verify(originalRoots[i], nID) { + if !row.verify(originalRoots[i], namespace) { return fmt.Errorf("row verification failed: row %d doesn't match original root: %s", i, root.String()) } } @@ -80,17 +77,18 @@ func (ns NamespacedShares) Verify(root *Root, nID namespace.ID) error { } // verify validates the row using nmt inclusion proof. -func (row *NamespacedRow) verify(rowRoot []byte, nID namespace.ID) bool { +func (row *NamespacedRow) verify(rowRoot []byte, namespace Namespace) bool { // construct nmt leaves from shares by prepending namespace leaves := make([][]byte, 0, len(row.Shares)) - for _, sh := range row.Shares { - leaves = append(leaves, append(sh[:NamespaceSize], sh...)) + for _, shr := range row.Shares { + leaves = append(leaves, append(GetNamespace(shr), shr...)) } // verify namespace return row.Proof.VerifyNamespace( sha256.New(), - nID, + namespace.ToNMT(), leaves, - rowRoot) + rowRoot, + ) } diff --git a/share/getters/cascade.go b/share/getters/cascade.go index 01540de926..fe99bed4c3 100644 --- a/share/getters/cascade.go +++ b/share/getters/cascade.go @@ -2,13 +2,11 @@ package getters import ( "context" - "encoding/hex" "errors" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/libs/utils" @@ -67,16 +65,16 @@ func (cg *CascadeGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d. func (cg *CascadeGetter) GetSharesByNamespace( ctx context.Context, root *share.Root, - id namespace.ID, + namespace share.Namespace, ) (share.NamespacedShares, error) { ctx, span := tracer.Start(ctx, "cascade/get-shares-by-namespace", trace.WithAttributes( attribute.String("root", root.String()), - attribute.String("nid", hex.EncodeToString(id)), + attribute.String("namespace", namespace.String()), )) defer span.End() get := func(ctx context.Context, get share.Getter) (share.NamespacedShares, error) { - return get.GetSharesByNamespace(ctx, root, id) + return get.GetSharesByNamespace(ctx, root, namespace) } return cascadeGetters(ctx, cg.getters, get) diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index 20b7e25191..7af8af2f26 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -14,12 +14,14 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/sharetest" ) func TestTeeGetter(t *testing.T) { @@ -39,8 +41,8 @@ func TestTeeGetter(t *testing.T) { tg := NewTeeGetter(ig, edsStore) t.Run("TeesToEDSStore", func(t *testing.T) { - eds, dah := randomEDS(t) - _, err := share.ImportShares(ctx, eds.Flattened(), bServ) + randEds, dah := randomEDS(t) + _, err := ipld.ImportShares(ctx, randEds.Flattened(), bServ) require.NoError(t, err) // eds store doesn't have the EDS yet @@ -50,7 +52,7 @@ func TestTeeGetter(t *testing.T) { retrievedEDS, err := tg.GetEDS(ctx, &dah) require.NoError(t, err) - require.True(t, share.EqualEDS(eds, retrievedEDS)) + require.True(t, share.EqualEDS(randEds, retrievedEDS)) // eds store now has the EDS and it can be retrieved ok, err = edsStore.Has(ctx, dah.Hash()) @@ -58,22 +60,22 @@ func TestTeeGetter(t *testing.T) { assert.NoError(t, err) finalEDS, err := edsStore.Get(ctx, dah.Hash()) assert.NoError(t, err) - require.True(t, share.EqualEDS(eds, finalEDS)) + require.True(t, share.EqualEDS(randEds, finalEDS)) }) t.Run("ShardAlreadyExistsDoesntError", func(t *testing.T) { - eds, dah := randomEDS(t) - _, err := share.ImportShares(ctx, eds.Flattened(), bServ) + randEds, dah := randomEDS(t) + _, err := ipld.ImportShares(ctx, randEds.Flattened(), bServ) require.NoError(t, err) retrievedEDS, err := tg.GetEDS(ctx, &dah) require.NoError(t, err) - require.True(t, share.EqualEDS(eds, retrievedEDS)) + require.True(t, share.EqualEDS(randEds, retrievedEDS)) // no error should be returned, even though the EDS identified by the DAH already exists retrievedEDS, err = tg.GetEDS(ctx, &dah) require.NoError(t, err) - require.True(t, share.EqualEDS(eds, retrievedEDS)) + require.True(t, share.EqualEDS(randEds, retrievedEDS)) }) } @@ -92,16 +94,16 @@ func TestStoreGetter(t *testing.T) { sg := NewStoreGetter(edsStore) t.Run("GetShare", func(t *testing.T) { - eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) + randEds, dah := randomEDS(t) + err = edsStore.Put(ctx, dah.Hash(), randEds) require.NoError(t, err) - squareSize := int(eds.Width()) + squareSize := int(randEds.Width()) for i := 0; i < squareSize; i++ { for j := 0; j < squareSize; j++ { share, err := sg.GetShare(ctx, &dah, i, j) require.NoError(t, err) - assert.Equal(t, eds.GetCell(uint(i), uint(j)), share) + assert.Equal(t, randEds.GetCell(uint(i), uint(j)), share) } } @@ -112,13 +114,13 @@ func TestStoreGetter(t *testing.T) { }) t.Run("GetEDS", func(t *testing.T) { - eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) + randEds, dah := randomEDS(t) + err = edsStore.Put(ctx, dah.Hash(), randEds) require.NoError(t, err) retrievedEDS, err := sg.GetEDS(ctx, &dah) require.NoError(t, err) - assert.True(t, share.EqualEDS(eds, retrievedEDS)) + assert.True(t, share.EqualEDS(randEds, retrievedEDS)) // root not found root := share.Root{} @@ -127,24 +129,24 @@ func TestStoreGetter(t *testing.T) { }) t.Run("GetSharesByNamespace", func(t *testing.T) { - eds, nID, dah := randomEDSWithDoubledNamespace(t, 4) - err = edsStore.Put(ctx, dah.Hash(), eds) + randEds, namespace, dah := randomEDSWithDoubledNamespace(t, 4) + err = edsStore.Put(ctx, dah.Hash(), randEds) require.NoError(t, err) - shares, err := sg.GetSharesByNamespace(ctx, &dah, nID) + shares, err := sg.GetSharesByNamespace(ctx, &dah, namespace) require.NoError(t, err) - require.NoError(t, shares.Verify(&dah, nID)) + require.NoError(t, shares.Verify(&dah, namespace)) assert.Len(t, shares.Flatten(), 2) - // nid not found - nID = make([]byte, namespace.NamespaceSize) - emptyShares, err := sg.GetSharesByNamespace(ctx, &dah, nID) + // namespace not found + randNamespace := sharetest.RandV0Namespace() + emptyShares, err := sg.GetSharesByNamespace(ctx, &dah, randNamespace) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) // root not found root := share.Root{} - _, err = sg.GetSharesByNamespace(ctx, &root, nID) + _, err = sg.GetSharesByNamespace(ctx, &root, namespace) require.ErrorIs(t, err, share.ErrNotFound) }) } @@ -168,16 +170,16 @@ func TestIPLDGetter(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) - eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) + randEds, dah := randomEDS(t) + err = edsStore.Put(ctx, dah.Hash(), randEds) require.NoError(t, err) - squareSize := int(eds.Width()) + squareSize := int(randEds.Width()) for i := 0; i < squareSize; i++ { for j := 0; j < squareSize; j++ { share, err := sg.GetShare(ctx, &dah, i, j) require.NoError(t, err) - assert.Equal(t, eds.GetCell(uint(i), uint(j)), share) + assert.Equal(t, randEds.GetCell(uint(i), uint(j)), share) } } @@ -191,47 +193,46 @@ func TestIPLDGetter(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) - eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) + randEds, dah := randomEDS(t) + err = edsStore.Put(ctx, dah.Hash(), randEds) require.NoError(t, err) retrievedEDS, err := sg.GetEDS(ctx, &dah) require.NoError(t, err) - assert.True(t, share.EqualEDS(eds, retrievedEDS)) + assert.True(t, share.EqualEDS(randEds, retrievedEDS)) }) t.Run("GetSharesByNamespace", func(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) - eds, nID, dah := randomEDSWithDoubledNamespace(t, 4) - err = edsStore.Put(ctx, dah.Hash(), eds) + randEds, namespace, dah := randomEDSWithDoubledNamespace(t, 4) + err = edsStore.Put(ctx, dah.Hash(), randEds) require.NoError(t, err) // first check that shares are returned correctly if they exist - shares, err := sg.GetSharesByNamespace(ctx, &dah, nID) + shares, err := sg.GetSharesByNamespace(ctx, &dah, namespace) require.NoError(t, err) - require.NoError(t, shares.Verify(&dah, nID)) + require.NoError(t, shares.Verify(&dah, namespace)) assert.Len(t, shares.Flatten(), 2) - // nid not found - nID = make([]byte, namespace.NamespaceSize) - emptyShares, err := sg.GetSharesByNamespace(ctx, &dah, nID) + // namespace not found + randNamespace := sharetest.RandV0Namespace() + emptyShares, err := sg.GetSharesByNamespace(ctx, &dah, randNamespace) require.NoError(t, err) - require.Nil(t, emptyShares) + require.Empty(t, emptyShares.Flatten()) // nid doesnt exist in root root := share.Root{} - emptyShares, err = sg.GetSharesByNamespace(ctx, &root, nID) + emptyShares, err = sg.GetSharesByNamespace(ctx, &root, namespace) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) }) } func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, share.Root) { - eds := share.RandEDS(t, 4) + eds := edstest.RandEDS(t, 4) dah := da.NewDataAvailabilityHeader(eds) - return eds, dah } @@ -239,7 +240,7 @@ func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, share.Root) { // middle that share a namespace. func randomEDSWithDoubledNamespace(t *testing.T, size int) (*rsmt2d.ExtendedDataSquare, []byte, share.Root) { n := size * size - randShares := share.RandShares(t, n) + randShares := sharetest.RandShares(t, n) idx1 := (n - 1) / 2 idx2 := n / 2 @@ -251,7 +252,7 @@ func randomEDSWithDoubledNamespace(t *testing.T, size int) (*rsmt2d.ExtendedData // D _ _ _ // _ _ _ _ // where the D shares have a common namespace. - copy(randShares[idx2][:share.NamespaceSize], randShares[idx1][:share.NamespaceSize]) + copy(share.GetNamespace(randShares[idx2]), share.GetNamespace(randShares[idx1])) eds, err := rsmt2d.ComputeExtendedDataSquare( randShares, @@ -261,5 +262,5 @@ func randomEDSWithDoubledNamespace(t *testing.T, size int) (*rsmt2d.ExtendedData require.NoError(t, err, "failure to recompute the extended data square") dah := da.NewDataAvailabilityHeader(eds) - return eds, randShares[idx1][:share.NamespaceSize], dah + return eds, share.GetNamespace(randShares[idx1]), dah } diff --git a/share/getters/ipld.go b/share/getters/ipld.go index 04a1f4f728..92d1c55678 100644 --- a/share/getters/ipld.go +++ b/share/getters/ipld.go @@ -2,7 +2,6 @@ package getters import ( "context" - "encoding/hex" "errors" "fmt" "sync" @@ -12,7 +11,6 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/libs/utils" @@ -55,7 +53,7 @@ func (ig *IPLDGetter) GetShare(ctx context.Context, dah *share.Root, row, col in // wrap the blockservice in a session if it has been signaled in the context. blockGetter := getGetter(ctx, ig.bServ) - s, err := share.GetShare(ctx, blockGetter, root, leaf, len(dah.RowRoots)) + s, err := ipld.GetShare(ctx, blockGetter, root, leaf, len(dah.RowRoots)) if errors.Is(err, ipld.ErrNodeNotFound) { // convert error to satisfy getter interface contract err = share.ErrNotFound @@ -90,24 +88,23 @@ func (ig *IPLDGetter) GetEDS(ctx context.Context, root *share.Root) (eds *rsmt2d func (ig *IPLDGetter) GetSharesByNamespace( ctx context.Context, root *share.Root, - nID namespace.ID, + namespace share.Namespace, ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "ipld/get-shares-by-namespace", trace.WithAttributes( attribute.String("root", root.String()), - attribute.String("nid", hex.EncodeToString(nID)), + attribute.String("namespace", namespace.String()), )) defer func() { utils.SetStatusAndEnd(span, err) }() - err = verifyNIDSize(nID) - if err != nil { - return nil, fmt.Errorf("getter/ipld: invalid namespace ID: %w", err) + if err = namespace.ValidateForData(); err != nil { + return nil, err } // wrap the blockservice in a session if it has been signaled in the context. blockGetter := getGetter(ctx, ig.bServ) - shares, err = collectSharesByNamespace(ctx, blockGetter, root, nID) + shares, err = collectSharesByNamespace(ctx, blockGetter, root, namespace) if errors.Is(err, ipld.ErrNodeNotFound) { // convert error to satisfy getter interface contract err = share.ErrNotFound diff --git a/share/getters/shrex.go b/share/getters/shrex.go index 99da2f8758..bb5e7ca7a7 100644 --- a/share/getters/shrex.go +++ b/share/getters/shrex.go @@ -2,7 +2,6 @@ package getters import ( "context" - "encoding/hex" "errors" "fmt" "time" @@ -12,10 +11,11 @@ import ( "go.opentelemetry.io/otel/metric/instrument" "go.opentelemetry.io/otel/metric/instrument/syncint64" "go.opentelemetry.io/otel/metric/unit" + "go.opentelemetry.io/otel/trace" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/p2p" "github.com/celestiaorg/celestia-node/share/p2p/peers" @@ -128,6 +128,13 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.Ex attempt int err error ) + ctx, span := tracer.Start(ctx, "shrex/get-eds", trace.WithAttributes( + attribute.String("root", root.String()), + )) + defer func() { + utils.SetStatusAndEnd(span, err) + }() + for { if ctx.Err() != nil { sg.metrics.recordEDSAttempt(ctx, attempt, false) @@ -181,15 +188,25 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.Ex func (sg *ShrexGetter) GetSharesByNamespace( ctx context.Context, root *share.Root, - id namespace.ID, + namespace share.Namespace, ) (share.NamespacedShares, error) { + if err := namespace.ValidateForData(); err != nil { + return nil, err + } var ( attempt int err error ) + ctx, span := tracer.Start(ctx, "shrex/get-shares-by-namespace", trace.WithAttributes( + attribute.String("root", root.String()), + attribute.String("namespace", namespace.String()), + )) + defer func() { + utils.SetStatusAndEnd(span, err) + }() // verify that the namespace could exist inside the roots before starting network requests - roots := filterRootsByNamespace(root, id) + roots := filterRootsByNamespace(root, namespace) if len(roots) == 0 { return nil, nil } @@ -205,7 +222,7 @@ func (sg *ShrexGetter) GetSharesByNamespace( if getErr != nil { log.Debugw("nd: couldn't find peer", "hash", root.String(), - "nid", hex.EncodeToString(id), + "namespace", namespace.String(), "err", getErr, "finished (s)", time.Since(start)) sg.metrics.recordNDAttempt(ctx, attempt, false) @@ -214,12 +231,12 @@ func (sg *ShrexGetter) GetSharesByNamespace( reqStart := time.Now() reqCtx, cancel := ctxWithSplitTimeout(ctx, sg.minAttemptsCount-attempt+1, sg.minRequestTimeout) - nd, getErr := sg.ndClient.RequestND(reqCtx, root, id, peer) + nd, getErr := sg.ndClient.RequestND(reqCtx, root, namespace, peer) cancel() switch { case getErr == nil: // both inclusion and non-inclusion cases needs verification - if verErr := nd.Verify(root, id); verErr != nil { + if verErr := nd.Verify(root, namespace); verErr != nil { getErr = verErr setStatus(peers.ResultBlacklistPeer) break @@ -244,7 +261,7 @@ func (sg *ShrexGetter) GetSharesByNamespace( } log.Debugw("nd: request failed", "hash", root.String(), - "nid", hex.EncodeToString(id), + "namespace", namespace.String(), "peer", peer.String(), "attempt", attempt, "err", getErr, diff --git a/share/getters/shrex_test.go b/share/getters/shrex_test.go index 72ac78e69b..e1628d8725 100644 --- a/share/getters/shrex_test.go +++ b/share/getters/shrex_test.go @@ -17,14 +17,13 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/nmt" - nmtnamespace "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/p2p/discovery" "github.com/celestiaorg/celestia-node/share/p2p/peers" "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" @@ -62,16 +61,16 @@ func TestShrexGetter(t *testing.T) { t.Cleanup(cancel) // generate test data - eds, dah, nID := generateTestEDS(t) - require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) + randEDS, dah, namespace := generateTestEDS(t) + require.NoError(t, edsStore.Put(ctx, dah.Hash(), randEDS)) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), Height: 1, }) - got, err := getter.GetSharesByNamespace(ctx, &dah, nID) + got, err := getter.GetSharesByNamespace(ctx, &dah, namespace) require.NoError(t, err) - require.NoError(t, got.Verify(&dah, nID)) + require.NoError(t, got.Verify(&dah, namespace)) }) t.Run("ND_err_not_found", func(t *testing.T) { @@ -79,13 +78,13 @@ func TestShrexGetter(t *testing.T) { t.Cleanup(cancel) // generate test data - _, dah, nID := generateTestEDS(t) + _, dah, namespace := generateTestEDS(t) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), Height: 1, }) - _, err := getter.GetSharesByNamespace(ctx, &dah, nID) + _, err := getter.GetSharesByNamespace(ctx, &dah, namespace) require.ErrorIs(t, err, share.ErrNotFound) }) @@ -94,7 +93,7 @@ func TestShrexGetter(t *testing.T) { t.Cleanup(cancel) // generate test data - eds, dah, maxNID := generateTestEDS(t) + eds, dah, maxNamespace := generateTestEDS(t) require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), @@ -102,11 +101,11 @@ func TestShrexGetter(t *testing.T) { }) // corrupt NID - nID := make([]byte, ipld.NamespaceSize) - copy(nID, maxNID) - nID[ipld.NamespaceSize-1]-- // pray for last byte to not be 0x00 + nID := make([]byte, share.NamespaceSize) + copy(nID, maxNamespace) + nID[share.NamespaceSize-1]-- // check for namespace to be between max and min namespace in root - require.Len(t, filterRootsByNamespace(&dah, maxNID), 1) + require.Len(t, filterRootsByNamespace(&dah, maxNamespace), 1) emptyShares, err := getter.GetSharesByNamespace(ctx, &dah, nID) require.NoError(t, err) @@ -120,25 +119,25 @@ func TestShrexGetter(t *testing.T) { t.Cleanup(cancel) // generate test data - eds, dah, maxNID := generateTestEDS(t) + eds, dah, maxNamesapce := generateTestEDS(t) require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), Height: 1, }) - // corrupt NID - nID := make([]byte, ipld.NamespaceSize) - copy(nID, maxNID) - nID[ipld.NamespaceSize-1]++ // pray for last byte to not be 0xFF + // corrupt namespace + namespace := make([]byte, share.NamespaceSize) + copy(namespace, maxNamesapce) + namespace[share.NamespaceSize-1]++ // check for namespace to be not in root - require.Len(t, filterRootsByNamespace(&dah, nID), 0) + require.Len(t, filterRootsByNamespace(&dah, namespace), 0) - emptyShares, err := getter.GetSharesByNamespace(ctx, &dah, nID) + emptyShares, err := getter.GetSharesByNamespace(ctx, &dah, namespace) require.NoError(t, err) // no shares should be returned require.Empty(t, emptyShares.Flatten()) - require.Nil(t, emptyShares.Verify(&dah, nID)) + require.Nil(t, emptyShares.Verify(&dah, namespace)) }) t.Run("EDS_Available", func(t *testing.T) { @@ -146,8 +145,8 @@ func TestShrexGetter(t *testing.T) { t.Cleanup(cancel) // generate test data - eds, dah, _ := generateTestEDS(t) - require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) + randEDS, dah, _ := generateTestEDS(t) + require.NoError(t, edsStore.Put(ctx, dah.Hash(), randEDS)) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), Height: 1, @@ -155,7 +154,7 @@ func TestShrexGetter(t *testing.T) { got, err := getter.GetEDS(ctx, &dah) require.NoError(t, err) - require.Equal(t, eds.Flattened(), got.Flattened()) + require.Equal(t, randEDS.Flattened(), got.Flattened()) }) t.Run("EDS_ctx_deadline", func(t *testing.T) { @@ -197,10 +196,10 @@ func newStore(t *testing.T) (*eds.Store, error) { return eds.NewStore(tmpDir, ds) } -func generateTestEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, da.DataAvailabilityHeader, nmtnamespace.ID) { - eds := share.RandEDS(t, 4) +func generateTestEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, da.DataAvailabilityHeader, share.Namespace) { + eds := edstest.RandEDS(t, 4) dah := da.NewDataAvailabilityHeader(eds) - max := nmt.MaxNamespace(dah.RowRoots[(len(dah.RowRoots))/2-1], ipld.NamespaceSize) + max := nmt.MaxNamespace(dah.RowRoots[(len(dah.RowRoots))/2-1], share.NamespaceSize) return eds, dah, max } diff --git a/share/getters/store.go b/share/getters/store.go index f39f41fc80..0b88d89155 100644 --- a/share/getters/store.go +++ b/share/getters/store.go @@ -2,14 +2,12 @@ package getters import ( "context" - "encoding/hex" "errors" "fmt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/libs/utils" @@ -58,7 +56,7 @@ func (sg *StoreGetter) GetShare(ctx context.Context, dah *share.Root, row, col i // wrap the read-only CAR blockstore in a getter blockGetter := eds.NewBlockGetter(bs) - s, err := share.GetShare(ctx, blockGetter, root, leaf, len(dah.RowRoots)) + s, err := ipld.GetShare(ctx, blockGetter, root, leaf, len(dah.RowRoots)) if errors.Is(err, ipld.ErrNodeNotFound) { // convert error to satisfy getter interface contract err = share.ErrNotFound @@ -95,19 +93,18 @@ func (sg *StoreGetter) GetEDS(ctx context.Context, root *share.Root) (data *rsmt func (sg *StoreGetter) GetSharesByNamespace( ctx context.Context, root *share.Root, - nID namespace.ID, + namespace share.Namespace, ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "store/get-shares-by-namespace", trace.WithAttributes( attribute.String("root", root.String()), - attribute.String("nid", hex.EncodeToString(nID)), + attribute.String("namespace", namespace.String()), )) defer func() { utils.SetStatusAndEnd(span, err) }() - err = verifyNIDSize(nID) - if err != nil { - return nil, fmt.Errorf("getter/store: invalid namespace ID: %w", err) + if err = namespace.ValidateForData(); err != nil { + return nil, err } bs, err := sg.store.CARBlockstore(ctx, root.Hash()) @@ -121,7 +118,7 @@ func (sg *StoreGetter) GetSharesByNamespace( // wrap the read-only CAR blockstore in a getter blockGetter := eds.NewBlockGetter(bs) - shares, err = collectSharesByNamespace(ctx, blockGetter, root, nID) + shares, err = collectSharesByNamespace(ctx, blockGetter, root, namespace) if err != nil { return nil, fmt.Errorf("getter/store: failed to retrieve shares by namespace: %w", err) } diff --git a/share/getters/tee.go b/share/getters/tee.go index 50e2e1b55d..213f05f90c 100644 --- a/share/getters/tee.go +++ b/share/getters/tee.go @@ -2,7 +2,6 @@ package getters import ( "context" - "encoding/hex" "errors" "fmt" @@ -10,7 +9,6 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/libs/utils" @@ -72,15 +70,15 @@ func (tg *TeeGetter) GetEDS(ctx context.Context, root *share.Root) (eds *rsmt2d. func (tg *TeeGetter) GetSharesByNamespace( ctx context.Context, root *share.Root, - id namespace.ID, + namespace share.Namespace, ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "tee/get-shares-by-namespace", trace.WithAttributes( attribute.String("root", root.String()), - attribute.String("nid", hex.EncodeToString(id)), + attribute.String("namespace", namespace.String()), )) defer func() { utils.SetStatusAndEnd(span, err) }() - return tg.getter.GetSharesByNamespace(ctx, root, id) + return tg.getter.GetSharesByNamespace(ctx, root, namespace) } diff --git a/share/getters/testing.go b/share/getters/testing.go index a90b937a51..4734557d7f 100644 --- a/share/getters/testing.go +++ b/share/getters/testing.go @@ -6,15 +6,15 @@ import ( "testing" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/edstest" ) // TestGetter provides a testing SingleEDSGetter and the root of the EDS it holds. func TestGetter(t *testing.T) (share.Getter, *share.Root) { - eds := share.RandEDS(t, 8) + eds := edstest.RandEDS(t, 8) dah := da.NewDataAvailabilityHeader(eds) return &SingleEDSGetter{ EDS: eds, @@ -46,7 +46,7 @@ func (seg *SingleEDSGetter) GetEDS(_ context.Context, root *share.Root) (*rsmt2d } // GetSharesByNamespace returns NamespacedShares from a kept EDS if the correct root is given. -func (seg *SingleEDSGetter) GetSharesByNamespace(context.Context, *share.Root, namespace.ID, +func (seg *SingleEDSGetter) GetSharesByNamespace(context.Context, *share.Root, share.Namespace, ) (share.NamespacedShares, error) { panic("SingleEDSGetter: GetSharesByNamespace is not implemented") } diff --git a/share/getters/utils.go b/share/getters/utils.go index 79438204bd..94b1cfe5ee 100644 --- a/share/getters/utils.go +++ b/share/getters/utils.go @@ -2,7 +2,6 @@ package getters import ( "context" - "encoding/hex" "errors" "fmt" "time" @@ -15,8 +14,6 @@ import ( "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" - "github.com/celestiaorg/nmt/namespace" - "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/ipld" @@ -30,34 +27,33 @@ var ( ) // filterRootsByNamespace returns the row roots from the given share.Root that contain the passed -// namespace ID. -func filterRootsByNamespace(root *share.Root, nID namespace.ID) []cid.Cid { +// namespace. +func filterRootsByNamespace(root *share.Root, namespace share.Namespace) []cid.Cid { rowRootCIDs := make([]cid.Cid, 0, len(root.RowRoots)) for _, row := range root.RowRoots { - if !ipld.NamespaceIsOutsideRange(row, row, nID) { + if !namespace.IsOutsideRange(row, row) { rowRootCIDs = append(rowRootCIDs, ipld.MustCidFromNamespacedSha256(row)) } } return rowRootCIDs } -// collectSharesByNamespace collects NamespaceShares within the given namespace ID from the given -// share.Root. +// collectSharesByNamespace collects NamespaceShares within the given namespace from share.Root. func collectSharesByNamespace( ctx context.Context, bg blockservice.BlockGetter, root *share.Root, - nID namespace.ID, + namespace share.Namespace, ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "collect-shares-by-namespace", trace.WithAttributes( attribute.String("root", root.String()), - attribute.String("nid", hex.EncodeToString(nID)), + attribute.String("namespace", namespace.String()), )) defer func() { utils.SetStatusAndEnd(span, err) }() - rootCIDs := filterRootsByNamespace(root, nID) + rootCIDs := filterRootsByNamespace(root, namespace) if len(rootCIDs) == 0 { return nil, nil } @@ -68,13 +64,13 @@ func collectSharesByNamespace( // shadow loop variables, to ensure correct values are captured i, rootCID := i, rootCID errGroup.Go(func() error { - row, proof, err := share.GetSharesByNamespace(ctx, bg, rootCID, nID, len(root.RowRoots)) + row, proof, err := ipld.GetSharesByNamespace(ctx, bg, rootCID, namespace, len(root.RowRoots)) shares[i] = share.NamespacedRow{ Shares: row, Proof: proof, } if err != nil { - return fmt.Errorf("retrieving nID %x for row %x: %w", nID, rootCID, err) + return fmt.Errorf("retrieving shares by namespace %s for row %x: %w", namespace.String(), rootCID, err) } return nil }) @@ -83,15 +79,8 @@ func collectSharesByNamespace( if err := errGroup.Wait(); err != nil { return nil, err } - return shares, nil -} -func verifyNIDSize(nID namespace.ID) error { - if len(nID) != share.NamespaceSize { - return fmt.Errorf("expected namespace ID of size %d, got %d", - share.NamespaceSize, len(nID)) - } - return nil + return shares, nil } // ctxWithSplitTimeout will split timeout stored in context by splitFactor and return the result if diff --git a/share/helpers.go b/share/helpers.go new file mode 100644 index 0000000000..0d729fd9b2 --- /dev/null +++ b/share/helpers.go @@ -0,0 +1,58 @@ +package share + +import ( + "bytes" + + "github.com/celestiaorg/rsmt2d" +) + +// TODO(Wondertan): All these helpers should be methods on rsmt2d.EDS + +// ExtractODS returns the original shares of the given ExtendedDataSquare. This +// is a helper function for circumstances where AddShares must be used after the EDS has already +// been generated. +func ExtractODS(eds *rsmt2d.ExtendedDataSquare) []Share { + origWidth := eds.Width() / 2 + origShares := make([][]byte, origWidth*origWidth) + for i := uint(0); i < origWidth; i++ { + row := eds.Row(i) + for j := uint(0); j < origWidth; j++ { + origShares[(i*origWidth)+j] = row[j] + } + } + return origShares +} + +// ExtractEDS takes an EDS and extracts all shares from it in a flattened slice(row by row). +func ExtractEDS(eds *rsmt2d.ExtendedDataSquare) []Share { + flattenedEDSSize := eds.Width() * eds.Width() + out := make([][]byte, flattenedEDSSize) + count := 0 + for i := uint(0); i < eds.Width(); i++ { + for _, share := range eds.Row(i) { + out[count] = share + count++ + } + } + return out +} + +// EqualEDS check whether two given EDSes are equal. +// TODO(Wondertan): Propose use of int by default instead of uint for the sake convenience and +// Golang practices +func EqualEDS(a *rsmt2d.ExtendedDataSquare, b *rsmt2d.ExtendedDataSquare) bool { + if a.Width() != b.Width() { + return false + } + + for i := uint(0); i < a.Width(); i++ { + ar, br := a.Row(i), b.Row(i) + for j := 0; j < len(ar); j++ { + if !bytes.Equal(ar[j], br[j]) { + return false + } + } + } + + return true +} diff --git a/share/add.go b/share/ipld/add.go similarity index 61% rename from share/add.go rename to share/ipld/add.go index 02016cadf6..fddfe8e99e 100644 --- a/share/add.go +++ b/share/ipld/add.go @@ -1,4 +1,4 @@ -package share +package ipld import ( "context" @@ -11,14 +11,14 @@ import ( "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/libs/utils" - "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share" ) // AddShares erasures and extends shares to blockservice.BlockService using the provided // ipld.NodeAdder. func AddShares( ctx context.Context, - shares []Share, + shares []share.Share, adder blockservice.BlockService, ) (*rsmt2d.ExtendedDataSquare, error) { if len(shares) == 0 { @@ -26,12 +26,12 @@ func AddShares( } squareSize := int(utils.SquareSize(len(shares))) // create nmt adder wrapping batch adder with calculated size - batchAdder := ipld.NewNmtNodeAdder(ctx, adder, ipld.MaxSizeBatchOption(squareSize*2)) + batchAdder := NewNmtNodeAdder(ctx, adder, MaxSizeBatchOption(squareSize*2)) // create the nmt wrapper to generate row and col commitments // recompute the eds eds, err := rsmt2d.ComputeExtendedDataSquare( shares, - DefaultRSMT2DCodec(), + share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(squareSize), nmt.NodeVisitor(batchAdder.Visit)), ) @@ -55,11 +55,11 @@ func ImportShares( } squareSize := int(utils.SquareSize(len(shares))) // create nmt adder wrapping batch adder with calculated size - batchAdder := ipld.NewNmtNodeAdder(ctx, adder, ipld.MaxSizeBatchOption(squareSize*2)) + batchAdder := NewNmtNodeAdder(ctx, adder, MaxSizeBatchOption(squareSize*2)) // recompute the eds eds, err := rsmt2d.ImportExtendedDataSquare( shares, - DefaultRSMT2DCodec(), + share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(squareSize/2), nmt.NodeVisitor(batchAdder.Visit)), ) @@ -72,31 +72,8 @@ func ImportShares( return eds, batchAdder.Commit() } -// ExtractODS returns the original shares of the given ExtendedDataSquare. This -// is a helper function for circumstances where AddShares must be used after the EDS has already -// been generated. -func ExtractODS(eds *rsmt2d.ExtendedDataSquare) []Share { - origWidth := eds.Width() / 2 - origShares := make([][]byte, origWidth*origWidth) - for i := uint(0); i < origWidth; i++ { - row := eds.Row(i) - for j := uint(0); j < origWidth; j++ { - origShares[(i*origWidth)+j] = row[j] - } - } - return origShares -} - -// ExtractEDS takes an EDS and extracts all shares from it in a flattened slice(row by row). -func ExtractEDS(eds *rsmt2d.ExtendedDataSquare) []Share { - flattenedEDSSize := eds.Width() * eds.Width() - out := make([][]byte, flattenedEDSSize) - count := 0 - for i := uint(0); i < eds.Width(); i++ { - for _, share := range eds.Row(i) { - out[count] = share - count++ - } - } - return out +func ImportEDS(ctx context.Context, square *rsmt2d.ExtendedDataSquare, adder blockservice.BlockService) error { + shares := share.ExtractEDS(square) + _, err := ImportShares(ctx, shares, adder) + return err } diff --git a/share/ipld/get.go b/share/ipld/get.go index 6d7de35c27..35f601853d 100644 --- a/share/ipld/get.go +++ b/share/ipld/get.go @@ -12,6 +12,8 @@ import ( ipld "github.com/ipfs/go-ipld-format" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" + + "github.com/celestiaorg/celestia-node/share" ) // NumWorkersLimit sets global limit for workers spawned by GetShares. @@ -26,7 +28,7 @@ import ( // // TODO(@Wondertan): This assumes we have parallelized DASer implemented. Sync the values once it is shipped. // TODO(@Wondertan): Allow configuration of values without global state. -var NumWorkersLimit = MaxSquareSize * MaxSquareSize / 2 * NumConcurrentSquares +var NumWorkersLimit = share.MaxSquareSize * share.MaxSquareSize / 2 * NumConcurrentSquares // NumConcurrentSquares limits the amount of squares that are fetched // concurrently/simultaneously. diff --git a/share/get.go b/share/ipld/get_shares.go similarity index 69% rename from share/get.go rename to share/ipld/get_shares.go index 9ec4c3c2c5..0bed240fdc 100644 --- a/share/get.go +++ b/share/ipld/get_shares.go @@ -1,4 +1,4 @@ -package share +package ipld import ( "context" @@ -8,9 +8,8 @@ import ( format "github.com/ipfs/go-ipld-format" "github.com/celestiaorg/nmt" - "github.com/celestiaorg/nmt/namespace" - "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share" ) // GetShare fetches and returns the data for leaf `leafIndex` of root `rootCid`. @@ -20,8 +19,8 @@ func GetShare( rootCid cid.Cid, leafIndex int, totalLeafs int, // this corresponds to the extended square width -) (Share, error) { - nd, err := ipld.GetLeaf(ctx, bGetter, rootCid, leafIndex, totalLeafs) +) (share.Share, error) { + nd, err := GetLeaf(ctx, bGetter, rootCid, leafIndex, totalLeafs) if err != nil { return nil, err } @@ -32,30 +31,30 @@ func GetShare( // GetShares walks the tree of a given root and puts shares into the given 'put' func. // Does not return any error, and returns/unblocks only on success // (got all shares) or on context cancellation. -func GetShares(ctx context.Context, bGetter blockservice.BlockGetter, root cid.Cid, shares int, put func(int, Share)) { +func GetShares(ctx context.Context, bg blockservice.BlockGetter, root cid.Cid, shares int, put func(int, share.Share)) { ctx, span := tracer.Start(ctx, "get-shares") defer span.End() putNode := func(i int, leaf format.Node) { put(i, leafToShare(leaf)) } - ipld.GetLeaves(ctx, bGetter, root, shares, putNode) + GetLeaves(ctx, bg, root, shares, putNode) } // GetSharesByNamespace walks the tree of a given root and returns its shares within the given -// namespace.ID. If a share could not be retrieved, err is not nil, and the returned array +// Namespace. If a share could not be retrieved, err is not nil, and the returned array // contains nil shares in place of the shares it was unable to retrieve. func GetSharesByNamespace( ctx context.Context, bGetter blockservice.BlockGetter, root cid.Cid, - nID namespace.ID, + namespace share.Namespace, maxShares int, -) ([]Share, *nmt.Proof, error) { +) ([]share.Share, *nmt.Proof, error) { ctx, span := tracer.Start(ctx, "get-shares-by-namespace") defer span.End() - data := ipld.NewNamespaceData(maxShares, nID, ipld.WithLeaves(), ipld.WithProofs()) + data := NewNamespaceData(maxShares, namespace, WithLeaves(), WithProofs()) err := data.CollectLeavesByNamespace(ctx, bGetter, root) if err != nil { return nil, nil, err @@ -63,7 +62,7 @@ func GetSharesByNamespace( leaves := data.Leaves() - shares := make([]Share, len(leaves)) + shares := make([]share.Share, len(leaves)) for i, leaf := range leaves { if leaf != nil { shares[i] = leafToShare(leaf) @@ -73,8 +72,8 @@ func GetSharesByNamespace( } // leafToShare converts an NMT leaf into a Share. -func leafToShare(nd format.Node) Share { +func leafToShare(nd format.Node) share.Share { // * Additional namespace is prepended so that parity data can be identified with a parity // namespace, which we cut off - return nd.RawData()[NamespaceSize:] + return share.GetData(nd.RawData()) } diff --git a/share/get_test.go b/share/ipld/get_shares_test.go similarity index 64% rename from share/get_test.go rename to share/ipld/get_shares_test.go index 1b9df69be3..cd26f759b3 100644 --- a/share/get_test.go +++ b/share/ipld/get_shares_test.go @@ -1,9 +1,8 @@ -package share +package ipld import ( "bytes" "context" - "crypto/rand" "errors" mrand "math/rand" "sort" @@ -23,11 +22,12 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/wrapper" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/libs/utils" - "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/sharetest" ) func TestGetShare(t *testing.T) { @@ -38,14 +38,14 @@ func TestGetShare(t *testing.T) { bServ := mdutils.Bserv() // generate random shares for the nmt - shares := RandShares(t, size*size) + shares := sharetest.RandShares(t, size*size) eds, err := AddShares(ctx, shares, bServ) require.NoError(t, err) for i, leaf := range shares { row := i / size pos := i - (size * row) - share, err := GetShare(ctx, bServ, ipld.MustCidFromNamespacedSha256(eds.RowRoots()[row]), pos, size*2) + share, err := GetShare(ctx, bServ, MustCidFromNamespacedSha256(eds.RowRoots()[row]), pos, size*2) require.NoError(t, err) assert.Equal(t, leaf, share) } @@ -58,12 +58,12 @@ func TestBlockRecovery(t *testing.T) { extendedShareCount := extendedSquareWidth * extendedSquareWidth // generate test data - quarterShares := RandShares(t, shareCount) - allShares := RandShares(t, shareCount) + quarterShares := sharetest.RandShares(t, shareCount) + allShares := sharetest.RandShares(t, shareCount) testCases := []struct { name string - shares []Share + shares []share.Share expectErr bool errString string d int // number of shares to delete @@ -79,25 +79,29 @@ func TestBlockRecovery(t *testing.T) { t.Run(tc.name, func(t *testing.T) { squareSize := utils.SquareSize(len(tc.shares)) - eds, err := rsmt2d.ComputeExtendedDataSquare(tc.shares, DefaultRSMT2DCodec(), wrapper.NewConstructor(squareSize)) + testEds, err := rsmt2d.ComputeExtendedDataSquare( + tc.shares, + share.DefaultRSMT2DCodec(), + wrapper.NewConstructor(squareSize), + ) require.NoError(t, err) // calculate roots using the first complete square - rowRoots := eds.RowRoots() - colRoots := eds.ColRoots() + rowRoots := testEds.RowRoots() + colRoots := testEds.ColRoots() - flat := ExtractEDS(eds) + flat := share.ExtractEDS(testEds) // recover a partially complete square rdata := removeRandShares(flat, tc.d) - eds, err = rsmt2d.ImportExtendedDataSquare( + testEds, err = rsmt2d.ImportExtendedDataSquare( rdata, - DefaultRSMT2DCodec(), + share.DefaultRSMT2DCodec(), wrapper.NewConstructor(squareSize), ) require.NoError(t, err) - err = eds.Repair(rowRoots, colRoots) + err = testEds.Repair(rowRoots, colRoots) if tc.expectErr { require.Error(t, err) require.Contains(t, err.Error(), tc.errString) @@ -105,27 +109,27 @@ func TestBlockRecovery(t *testing.T) { } assert.NoError(t, err) - reds, err := rsmt2d.ImportExtendedDataSquare(rdata, DefaultRSMT2DCodec(), wrapper.NewConstructor(squareSize)) + reds, err := rsmt2d.ImportExtendedDataSquare(rdata, share.DefaultRSMT2DCodec(), wrapper.NewConstructor(squareSize)) require.NoError(t, err) // check that the squares are equal - assert.Equal(t, ExtractEDS(eds), ExtractEDS(reds)) + assert.Equal(t, share.ExtractEDS(testEds), share.ExtractEDS(reds)) }) } } func Test_ConvertEDStoShares(t *testing.T) { squareWidth := 16 - shares := RandShares(t, squareWidth*squareWidth) + shares := sharetest.RandShares(t, squareWidth*squareWidth) // compute extended square - eds, err := rsmt2d.ComputeExtendedDataSquare( + testEds, err := rsmt2d.ComputeExtendedDataSquare( shares, - DefaultRSMT2DCodec(), + share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(squareWidth)), ) require.NoError(t, err) - resshares := ExtractODS(eds) + resshares := share.ExtractODS(testEds) require.Equal(t, shares, resshares) } @@ -150,29 +154,29 @@ func TestGetSharesByNamespace(t *testing.T) { bServ := mdutils.Bserv() var tests = []struct { - rawData []Share + rawData []share.Share }{ - {rawData: RandShares(t, 4)}, - {rawData: RandShares(t, 16)}, + {rawData: sharetest.RandShares(t, 4)}, + {rawData: sharetest.RandShares(t, 16)}, } for i, tt := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { - // choose random nID from rand shares + // choose random namespace from rand shares expected := tt.rawData[len(tt.rawData)/2] - nID := expected[:NamespaceSize] + namespace := share.GetNamespace(expected) - // change rawData to contain several shares with same nID + // change rawData to contain several shares with same namespace tt.rawData[(len(tt.rawData)/2)+1] = expected // put raw data in BlockService eds, err := AddShares(ctx, tt.rawData, bServ) require.NoError(t, err) - var shares []Share + var shares []share.Share for _, row := range eds.RowRoots() { - rcid := ipld.MustCidFromNamespacedSha256(row) - rowShares, _, err := GetSharesByNamespace(ctx, bServ, rcid, nID, len(eds.RowRoots())) - if errors.Is(err, ipld.ErrNamespaceOutsideRange) { + rcid := MustCidFromNamespacedSha256(row) + rowShares, _, err := GetSharesByNamespace(ctx, bServ, rcid, namespace, len(eds.RowRoots())) + if errors.Is(err, ErrNamespaceOutsideRange) { continue } require.NoError(t, err) @@ -193,13 +197,12 @@ func TestCollectLeavesByNamespace_IncompleteData(t *testing.T) { t.Cleanup(cancel) bServ := mdutils.Bserv() - shares := RandShares(t, 16) + shares := sharetest.RandShares(t, 16) // set all shares to the same namespace id - nid := shares[0][:NamespaceSize] - - for _, nspace := range shares { - copy(nspace[:NamespaceSize], nid) + namespace := share.GetNamespace(shares[0]) + for _, shr := range shares { + copy(share.GetNamespace(shr), namespace) } eds, err := AddShares(ctx, shares, bServ) @@ -208,28 +211,28 @@ func TestCollectLeavesByNamespace_IncompleteData(t *testing.T) { roots := eds.RowRoots() // remove the second share from the first row - rcid := ipld.MustCidFromNamespacedSha256(roots[0]) - node, err := ipld.GetNode(ctx, bServ, rcid) + rcid := MustCidFromNamespacedSha256(roots[0]) + node, err := GetNode(ctx, bServ, rcid) require.NoError(t, err) // Left side of the tree contains the original shares - data, err := ipld.GetNode(ctx, bServ, node.Links()[0].Cid) + data, err := GetNode(ctx, bServ, node.Links()[0].Cid) require.NoError(t, err) // Second share is the left side's right child - l, err := ipld.GetNode(ctx, bServ, data.Links()[0].Cid) + l, err := GetNode(ctx, bServ, data.Links()[0].Cid) require.NoError(t, err) - r, err := ipld.GetNode(ctx, bServ, l.Links()[1].Cid) + r, err := GetNode(ctx, bServ, l.Links()[1].Cid) require.NoError(t, err) err = bServ.DeleteBlock(ctx, r.Cid()) require.NoError(t, err) - namespaceData := ipld.NewNamespaceData(len(shares), nid, ipld.WithLeaves()) + namespaceData := NewNamespaceData(len(shares), namespace, WithLeaves()) err = namespaceData.CollectLeavesByNamespace(ctx, bServ, rcid) + require.Error(t, err) leaves := namespaceData.Leaves() assert.Nil(t, leaves[1]) assert.Equal(t, 4, len(leaves)) - require.Error(t, err) } func TestCollectLeavesByNamespace_AbsentNamespaceId(t *testing.T) { @@ -237,42 +240,42 @@ func TestCollectLeavesByNamespace_AbsentNamespaceId(t *testing.T) { t.Cleanup(cancel) bServ := mdutils.Bserv() - shares := RandShares(t, 1024) + shares := sharetest.RandShares(t, 1024) - // set all shares to the same namespace id - nids, err := randomNids(5) + // set all shares to the same namespace + namespaces, err := randomNamespaces(5) require.NoError(t, err) - minNid := nids[0] - minIncluded := nids[1] - midNid := nids[2] - maxIncluded := nids[3] - maxNid := nids[4] + minNamespace := namespaces[0] + minIncluded := namespaces[1] + midNamespace := namespaces[2] + maxIncluded := namespaces[3] + maxNamespace := namespaces[4] secondNamespaceFrom := mrand.Intn(len(shares)-2) + 1 - for i, nspace := range shares { + for i, shr := range shares { if i < secondNamespaceFrom { - copy(nspace[:NamespaceSize], minIncluded) + copy(share.GetNamespace(shr), minIncluded) continue } - copy(nspace[:NamespaceSize], maxIncluded) + copy(share.GetNamespace(shr), maxIncluded) } var tests = []struct { - name string - data []Share - missingNid []byte - isAbsence bool + name string + data []share.Share + missingNamespace share.Namespace + isAbsence bool }{ - {name: "Namespace id less than the minimum namespace in data", data: shares, missingNid: minNid}, - {name: "Namespace id greater than the maximum namespace in data", data: shares, missingNid: maxNid}, - {name: "Namespace id in range but still missing", data: shares, missingNid: midNid, isAbsence: true}, + {name: "Namespace less than the minimum namespace in data", data: shares, missingNamespace: minNamespace}, + {name: "Namespace greater than the maximum namespace in data", data: shares, missingNamespace: maxNamespace}, + {name: "Namespace in range but still missing", data: shares, missingNamespace: midNamespace, isAbsence: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { eds, err := AddShares(ctx, shares, bServ) require.NoError(t, err) - assertNoRowContainsNID(ctx, t, bServ, eds, tt.missingNid, tt.isAbsence) + assertNoRowContainsNID(ctx, t, bServ, eds, tt.missingNamespace, tt.isAbsence) }) } } @@ -282,10 +285,10 @@ func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testi t.Cleanup(cancel) bServ := mdutils.Bserv() - shares := RandShares(t, 16) + shares := sharetest.RandShares(t, 16) // set all shares to the same namespace and data but the last one - nid := shares[0][:NamespaceSize] + namespace := share.GetNamespace(shares[0]) commonNamespaceData := shares[0] for i, nspace := range shares { @@ -300,10 +303,10 @@ func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testi require.NoError(t, err) for _, row := range eds.RowRoots() { - rcid := ipld.MustCidFromNamespacedSha256(row) - data := ipld.NewNamespaceData(len(shares), nid, ipld.WithLeaves()) + rcid := MustCidFromNamespacedSha256(row) + data := NewNamespaceData(len(shares), namespace, WithLeaves()) err := data.CollectLeavesByNamespace(ctx, bServ, rcid) - if errors.Is(err, ipld.ErrNamespaceOutsideRange) { + if errors.Is(err, ErrNamespaceOutsideRange) { continue } assert.Nil(t, err) @@ -311,7 +314,7 @@ func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testi for _, node := range leaves { // test that the data returned by collectLeavesByNamespace for nid // matches the commonNamespaceData that was copied across almost all data - assert.Equal(t, commonNamespaceData, node.RawData()[NamespaceSize:]) + assert.Equal(t, commonNamespaceData, share.GetData(node.RawData())) } } } @@ -322,11 +325,11 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { bServ := mdutils.Bserv() var tests = []struct { - rawData []Share + rawData []share.Share }{ - {rawData: RandShares(t, 4)}, - {rawData: RandShares(t, 16)}, - {rawData: RandShares(t, 64)}, + {rawData: sharetest.RandShares(t, 4)}, + {rawData: sharetest.RandShares(t, 16)}, + {rawData: sharetest.RandShares(t, 64)}, } for i, tt := range tests { @@ -341,9 +344,9 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { } expected := tt.rawData[from] - nID := namespace.ID(expected[:NamespaceSize]) + namespace := share.GetNamespace(expected) - // change rawData to contain several shares with same nID + // change rawData to contain several shares with same namespace for i := from; i <= to; i++ { tt.rawData[i] = expected } @@ -352,12 +355,12 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { eds, err := AddShares(ctx, tt.rawData, bServ) require.NoError(t, err) - var shares []Share + var shares []share.Share for _, row := range eds.RowRoots() { - rcid := ipld.MustCidFromNamespacedSha256(row) - rowShares, proof, err := GetSharesByNamespace(ctx, bServ, rcid, nID, len(eds.RowRoots())) - if ipld.NamespaceIsOutsideRange(row, row, nID) { - require.ErrorIs(t, err, ipld.ErrNamespaceOutsideRange) + rcid := MustCidFromNamespacedSha256(row) + rowShares, proof, err := GetSharesByNamespace(ctx, bServ, rcid, namespace, len(eds.RowRoots())) + if namespace.IsOutsideRange(row, row) { + require.ErrorIs(t, err, ErrNamespaceOutsideRange) continue } require.NoError(t, err) @@ -368,24 +371,24 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { // construct nodes from shares by prepending namespace var leaves [][]byte - for _, sh := range rowShares { - leaves = append(leaves, append(sh[:NamespaceSize], sh...)) + for _, shr := range rowShares { + leaves = append(leaves, append(share.GetNamespace(shr), shr...)) } // verify namespace verified := proof.VerifyNamespace( sha256.New(), - nID, + namespace.ToNMT(), leaves, - ipld.NamespacedSha256FromCID(rcid)) + NamespacedSha256FromCID(rcid)) require.True(t, verified) // verify inclusion verified = proof.VerifyInclusion( sha256.New(), - nID, + namespace.ToNMT(), rowShares, - ipld.NamespacedSha256FromCID(rcid)) + NamespacedSha256FromCID(rcid)) require.True(t, verified) } } @@ -409,7 +412,7 @@ func TestBatchSize(t *testing.T) { {"8", 8}, {"16", 16}, {"32", 32}, - // {"64", 64}, // test case too large for CI with race detector + {"64", 64}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -418,8 +421,8 @@ func TestBatchSize(t *testing.T) { bs := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) - eds := RandEDS(t, tt.origWidth) - _, err := AddShares(ctx, ExtractODS(eds), blockservice.New(bs, offline.Exchange(bs))) + randEds := edstest.RandEDS(t, tt.origWidth) + _, err := AddShares(ctx, share.ExtractODS(randEds), blockservice.New(bs, offline.Exchange(bs))) require.NoError(t, err) out, err := bs.AllKeysChan(ctx) @@ -430,7 +433,7 @@ func TestBatchSize(t *testing.T) { count++ } extendedWidth := tt.origWidth * 2 - assert.Equalf(t, count, ipld.BatchSize(extendedWidth), "batchSize(%v)", extendedWidth) + assert.Equalf(t, count, BatchSize(extendedWidth), "batchSize(%v)", extendedWidth) }) } } @@ -440,57 +443,52 @@ func assertNoRowContainsNID( t *testing.T, bServ blockservice.BlockService, eds *rsmt2d.ExtendedDataSquare, - nID namespace.ID, + namespace share.Namespace, isAbsent bool, ) { rowRootCount := len(eds.RowRoots()) // get all row root cids rowRootCIDs := make([]cid.Cid, rowRootCount) for i, rowRoot := range eds.RowRoots() { - rowRootCIDs[i] = ipld.MustCidFromNamespacedSha256(rowRoot) + rowRootCIDs[i] = MustCidFromNamespacedSha256(rowRoot) } - // for each row root cid check if the minNID exists + // for each row root cid check if the min namespace exists var absentCount, foundAbsenceRows int for _, rowRoot := range eds.RowRoots() { var outsideRange bool - if !ipld.NamespaceIsOutsideRange(rowRoot, rowRoot, nID) { - // nID does belong to namespace range of the row + if !namespace.IsOutsideRange(rowRoot, rowRoot) { + // namespace does belong to namespace range of the row absentCount++ } else { outsideRange = true } - data := ipld.NewNamespaceData(rowRootCount, nID, ipld.WithProofs()) - rootCID := ipld.MustCidFromNamespacedSha256(rowRoot) + data := NewNamespaceData(rowRootCount, namespace, WithProofs()) + rootCID := MustCidFromNamespacedSha256(rowRoot) err := data.CollectLeavesByNamespace(ctx, bServ, rootCID) if outsideRange { - require.ErrorIs(t, err, ipld.ErrNamespaceOutsideRange) + require.ErrorIs(t, err, ErrNamespaceOutsideRange) continue } require.NoError(t, err) // if no error returned, check absence proof foundAbsenceRows++ - verified := data.Proof().VerifyNamespace(sha256.New(), nID, nil, rowRoot) + verified := data.Proof().VerifyNamespace(sha256.New(), namespace.ToNMT(), nil, rowRoot) require.True(t, verified) } if isAbsent { require.Equal(t, foundAbsenceRows, absentCount) - // there should be max 1 row that has namespace range containing nID + // there should be max 1 row that has namespace range containing namespace require.LessOrEqual(t, absentCount, 1) } } -func randomNids(total int) ([]namespace.ID, error) { - namespaces := make([]namespace.ID, total) +func randomNamespaces(total int) ([]share.Namespace, error) { + namespaces := make([]share.Namespace, total) for i := range namespaces { - nid := make([]byte, NamespaceSize) - _, err := rand.Read(nid) - if err != nil { - return nil, err - } - namespaces[i] = nid + namespaces[i] = sharetest.RandV0Namespace() } sort.Slice(namespaces, func(i, j int) bool { return bytes.Compare(namespaces[i], namespaces[j]) < 0 }) return namespaces, nil diff --git a/share/ipld/namespace_data.go b/share/ipld/namespace_data.go index f86a9803ac..d776da219d 100644 --- a/share/ipld/namespace_data.go +++ b/share/ipld/namespace_data.go @@ -2,7 +2,6 @@ package ipld import ( "context" - "encoding/hex" "errors" "fmt" "sync" @@ -15,11 +14,12 @@ import ( "go.opentelemetry.io/otel/codes" "github.com/celestiaorg/nmt" - "github.com/celestiaorg/nmt/namespace" + + "github.com/celestiaorg/celestia-node/share" ) var ErrNamespaceOutsideRange = errors.New("share/ipld: " + - "target namespace id is outside of namespace range for the given root") + "target namespace is outside of namespace range for the given root") // Option is the functional option that is applied to the NamespaceData instance // to configure data that needs to be stored. @@ -48,20 +48,20 @@ type NamespaceData struct { bounds fetchedBounds maxShares int - nID namespace.ID + namespace share.Namespace isAbsentNamespace atomic.Bool absenceProofLeaf ipld.Node } -func NewNamespaceData(maxShares int, nID namespace.ID, options ...Option) *NamespaceData { +func NewNamespaceData(maxShares int, namespace share.Namespace, options ...Option) *NamespaceData { data := &NamespaceData{ // we don't know where in the tree the leaves in the namespace are, // so we keep track of the bounds to return the correct slice // maxShares acts as a sentinel to know if we find any leaves bounds: fetchedBounds{int64(maxShares), 0}, maxShares: maxShares, - nID: nID, + namespace: namespace, } for _, opt := range options { @@ -71,8 +71,8 @@ func NewNamespaceData(maxShares int, nID namespace.ID, options ...Option) *Names } func (n *NamespaceData) validate(rootCid cid.Cid) error { - if len(n.nID) != NamespaceSize { - return fmt.Errorf("expected namespace ID of size %d, got %d", NamespaceSize, len(n.nID)) + if err := n.namespace.Validate(); err != nil { + return err } if n.leaves == nil && n.proofs == nil { @@ -80,7 +80,7 @@ func (n *NamespaceData) validate(rootCid cid.Cid) error { } root := NamespacedSha256FromCID(rootCid) - if NamespaceIsOutsideRange(root, root, n.nID) { + if n.namespace.IsOutsideRange(root, root) { return ErrNamespaceOutsideRange } return nil @@ -180,7 +180,7 @@ func (n *NamespaceData) Proof() *nmt.Proof { } // CollectLeavesByNamespace collects leaves and corresponding proof that could be used to verify -// leaves inclusion. It returns as many leaves from the given root with the given namespace.ID as +// leaves inclusion. It returns as many leaves from the given root with the given Namespace as // it can retrieve. If no shares are found, it returns error as nil. A // non-nil error means that only partial data is returned, because at least one share retrieval // failed. The following implementation is based on `GetShares`. @@ -197,7 +197,7 @@ func (n *NamespaceData) CollectLeavesByNamespace( defer span.End() span.SetAttributes( - attribute.String("namespace", hex.EncodeToString(n.nID)), + attribute.String("namespace", n.namespace.String()), attribute.String("root", root.String()), ) @@ -245,7 +245,7 @@ func (n *NamespaceData) CollectLeavesByNamespace( retrievalErr = err }) log.Errorw("could not retrieve IPLD node", - "nID", hex.EncodeToString(n.nID), + "namespace", n.namespace.String(), "pos", j.sharePos, "err", err, ) @@ -301,17 +301,17 @@ func (n *NamespaceData) collectNDWithProofs(j job, links []*ipld.Link) []job { var nextJobs []job // check if target namespace is outside of boundaries of both links - if NamespaceIsOutsideRange(leftLink, rightLink, n.nID) { + if n.namespace.IsOutsideRange(leftLink, rightLink) { log.Fatalf("target namespace outside of boundaries of links at depth: %v", j.depth) } - if !NamespaceIsAboveMax(leftLink, n.nID) { + if !n.namespace.IsAboveMax(leftLink) { // namespace is within the range of left link nextJobs = append(nextJobs, j.next(left, leftCid, false)) } else { - // proof is on the left side, if the nID is on the right side of the range of left link + // proof is on the left side, if the namespace is on the right side of the range of left link n.addProof(left, leftCid, j.depth) - if NamespaceIsBelowMin(rightLink, n.nID) { + if n.namespace.IsBelowMin(rightLink) { // namespace is not included in either links, convert to absence collector n.isAbsentNamespace.Store(true) nextJobs = append(nextJobs, j.next(right, rightCid, true)) @@ -319,11 +319,11 @@ func (n *NamespaceData) collectNDWithProofs(j job, links []*ipld.Link) []job { } } - if !NamespaceIsBelowMin(rightLink, n.nID) { + if !n.namespace.IsBelowMin(rightLink) { // namespace is within the range of right link nextJobs = append(nextJobs, j.next(right, rightCid, false)) } else { - // proof is on the right side, if the nID is on the left side of the range of right link + // proof is on the right side, if the namespace is on the left side of the range of right link n.addProof(right, rightCid, j.depth) } return nextJobs diff --git a/share/ipld/nmt.go b/share/ipld/nmt.go index df140ef8c7..485642dde4 100644 --- a/share/ipld/nmt.go +++ b/share/ipld/nmt.go @@ -11,7 +11,7 @@ import ( "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - blocks "github.com/ipfs/go-libipfs/blocks" + "github.com/ipfs/go-libipfs/blocks" logging "github.com/ipfs/go-log/v2" mh "github.com/multiformats/go-multihash" mhcore "github.com/multiformats/go-multihash/core" @@ -20,7 +20,8 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/nmt" - "github.com/celestiaorg/nmt/namespace" + + "github.com/celestiaorg/celestia-node/share" ) var ( @@ -39,17 +40,14 @@ const ( // that contain an NMT node (inner and leaf nodes). sha256NamespaceFlagged = 0x7701 - // NamespaceSize is a system-wide size for NMT namespaces. - NamespaceSize = appconsts.NamespaceSize - // NmtHashSize is the size of a digest created by an NMT in bytes. - NmtHashSize = 2*NamespaceSize + sha256.Size + NmtHashSize = 2*share.NamespaceSize + sha256.Size // innerNodeSize is the size of data in inner nodes. innerNodeSize = NmtHashSize * 2 // leafNodeSize is the size of data in leaf nodes. - leafNodeSize = NamespaceSize + appconsts.ShareSize + leafNodeSize = share.NamespaceSize + appconsts.ShareSize // cidPrefixSize is the size of the prepended buffer of the CID encoding // for NamespacedSha256. For more information, see: @@ -57,21 +55,15 @@ const ( cidPrefixSize = 4 // NMTIgnoreMaxNamespace is currently used value for IgnoreMaxNamespace option in NMT. - // IgnoreMaxNamespace defines whether the largest possible namespace.ID MAX_NID should be 'ignored'. + // IgnoreMaxNamespace defines whether the largest possible Namespace MAX_NID should be 'ignored'. // If set to true, this allows for shorter proofs in particular use-cases. NMTIgnoreMaxNamespace = true ) -var ( - // MaxSquareSize is currently the maximum size supported for unerasured data in - // rsmt2d.ExtendedDataSquare. - MaxSquareSize = appconsts.SquareSizeUpperBound(appconsts.LatestVersion) -) - func init() { // required for Bitswap to hash and verify inbound data correctly mhcore.Register(sha256NamespaceFlagged, func() hash.Hash { - nh := nmt.NewNmtHasher(sha256.New(), NamespaceSize, true) + nh := nmt.NewNmtHasher(sha256.New(), share.NamespaceSize, true) nh.Reset() return nh }) @@ -182,21 +174,3 @@ func Translate(dah *da.DataAvailabilityHeader, row, col int) (cid.Cid, int) { func NamespacedSha256FromCID(cid cid.Cid) []byte { return cid.Hash()[cidPrefixSize:] } - -// NamespaceIsAboveMax checks if the target namespace is above the maximum namespace for a given -// node hash. -func NamespaceIsAboveMax(nodeHash []byte, target namespace.ID) bool { - return !target.LessOrEqual(nmt.MaxNamespace(nodeHash, target.Size())) -} - -// NamespaceIsBelowMin checks if the target namespace is below the minimum namespace for a given -// node hash. -func NamespaceIsBelowMin(nodeHash []byte, target namespace.ID) bool { - return target.Less(nmt.MinNamespace(nodeHash, target.Size())) -} - -// NamespaceIsOutsideRange checks if the target namespace is outside the range defined by the left -// and right nodes -func NamespaceIsOutsideRange(leftNodeHash, rightNodeHash []byte, target namespace.ID) bool { - return NamespaceIsBelowMin(leftNodeHash, target) || NamespaceIsAboveMax(rightNodeHash, target) -} diff --git a/share/ipld/nmt_test.go b/share/ipld/nmt_test.go index b52a75c150..aa125ab3c7 100644 --- a/share/ipld/nmt_test.go +++ b/share/ipld/nmt_test.go @@ -1,36 +1,32 @@ package ipld import ( - "bytes" - "crypto/rand" - "sort" "strconv" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share/eds/edstest" ) // TestNamespaceFromCID checks that deriving the Namespaced hash from // the given CID works correctly. func TestNamespaceFromCID(t *testing.T) { var tests = []struct { - randData [][]byte + eds *rsmt2d.ExtendedDataSquare }{ // note that the number of shares must be a power of two - {randData: generateRandNamespacedRawData(4, appconsts.NamespaceSize, appconsts.ShareSize-appconsts.NamespaceSize)}, - {randData: generateRandNamespacedRawData(16, appconsts.NamespaceSize, appconsts.ShareSize-appconsts.NamespaceSize)}, + {eds: edstest.RandEDS(t, 4)}, + {eds: edstest.RandEDS(t, 16)}, } for i, tt := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { - // create DAH from rand data - eds, err := da.ExtendShares(tt.randData) - require.NoError(t, err) - dah := da.NewDataAvailabilityHeader(eds) + dah := da.NewDataAvailabilityHeader(tt.eds) // check to make sure NamespacedHash is correctly derived from CID for _, row := range dah.RowRoots { c, err := CidFromNamespacedSha256(row) @@ -42,28 +38,3 @@ func TestNamespaceFromCID(t *testing.T) { }) } } - -// generateRandNamespacedRawData returns random namespaced raw data for testing -// purposes. Note that this does not check that total is a power of two. -func generateRandNamespacedRawData(total, nidSize, leafSize uint32) [][]byte { - data := make([][]byte, total) - for i := uint32(0); i < total; i++ { - nid := make([]byte, nidSize) - - _, _ = rand.Read(nid) - data[i] = nid - } - sortByteArrays(data) - for i := uint32(0); i < total; i++ { - d := make([]byte, leafSize) - - _, _ = rand.Read(d) - data[i] = append(data[i], d...) - } - - return data -} - -func sortByteArrays(src [][]byte) { - sort.Slice(src, func(i, j int) bool { return bytes.Compare(src[i], src[j]) < 0 }) -} diff --git a/share/mocks/getter.go b/share/mocks/getter.go index 1c73c9170d..2a1b84efd5 100644 --- a/share/mocks/getter.go +++ b/share/mocks/getter.go @@ -8,11 +8,11 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + da "github.com/celestiaorg/celestia-app/pkg/da" share "github.com/celestiaorg/celestia-node/share" - namespace "github.com/celestiaorg/nmt/namespace" rsmt2d "github.com/celestiaorg/rsmt2d" - gomock "github.com/golang/mock/gomock" ) // MockGetter is a mock of Getter interface. @@ -69,7 +69,7 @@ func (mr *MockGetterMockRecorder) GetShare(arg0, arg1, arg2, arg3 interface{}) * } // GetSharesByNamespace mocks base method. -func (m *MockGetter) GetSharesByNamespace(arg0 context.Context, arg1 *da.DataAvailabilityHeader, arg2 namespace.ID) (share.NamespacedShares, error) { +func (m *MockGetter) GetSharesByNamespace(arg0 context.Context, arg1 *da.DataAvailabilityHeader, arg2 share.Namespace) (share.NamespacedShares, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSharesByNamespace", arg0, arg1, arg2) ret0, _ := ret[0].(share.NamespacedShares) diff --git a/share/namespace.go b/share/namespace.go index 0af1d1a60e..433529a57f 100644 --- a/share/namespace.go +++ b/share/namespace.go @@ -9,6 +9,9 @@ import ( "github.com/celestiaorg/nmt/namespace" ) +// NamespaceSize is a system-wide size for NMT namespaces. +const NamespaceSize = appns.NamespaceSize + // Various reserved namespaces. var ( MaxReservedNamespace = Namespace(appns.MaxReservedNamespace.Bytes()) @@ -17,12 +20,30 @@ var ( ReservedPaddingNamespace = Namespace(appns.ReservedPaddingNamespace.Bytes()) TxNamespace = Namespace(appns.TxNamespace.Bytes()) PayForBlobNamespace = Namespace(appns.PayForBlobNamespace.Bytes()) + ISRNamespace = Namespace(appns.IntermediateStateRootsNamespace.Bytes()) ) // Namespace represents namespace of a Share. // Consists of version byte and namespace ID. type Namespace []byte +// NewBlobNamespaceV0 takes a variable size byte slice and creates a valid version 0 Blob Namespace. +// The byte slice must be <= 10 bytes. +// If it is less than 10 bytes, it will be left padded to size 10 with 0s. +// Use predefined namespaces above, if non-blob namespace is needed. +func NewBlobNamespaceV0(id []byte) (Namespace, error) { + if len(id) == 0 || len(id) > appns.NamespaceVersionZeroIDSize { + return nil, fmt.Errorf( + "namespace id must be > 0 && <= %d, but it was %d bytes", appns.NamespaceVersionZeroIDSize, len(id)) + } + + n := make(Namespace, NamespaceSize) + // version and zero padding are already set as zero, + // so simply copying subNID to the end is enough to comply the V0 spec + copy(n[len(n)-len(id):], id) + return n, n.ValidateForBlob() +} + // NamespaceFromBytes converts bytes into Namespace and validates it. func NamespaceFromBytes(b []byte) (Namespace, error) { n := Namespace(b) @@ -84,8 +105,8 @@ func (n Namespace) Validate() error { return nil } -// ValidateDataNamespace checks if the Namespace contains real/useful data. -func (n Namespace) ValidateDataNamespace() error { +// ValidateForData checks if the Namespace is of real/useful data. +func (n Namespace) ValidateForData() error { if err := n.Validate(); err != nil { return err } @@ -95,9 +116,9 @@ func (n Namespace) ValidateDataNamespace() error { return nil } -// ValidateBlobNamespace checks if the Namespace is valid blob namespace. -func (n Namespace) ValidateBlobNamespace() error { - if err := n.ValidateDataNamespace(); err != nil { +// ValidateForBlob checks if the Namespace is valid blob namespace. +func (n Namespace) ValidateForBlob() error { + if err := n.ValidateForData(); err != nil { return err } if bytes.Compare(n, MaxReservedNamespace) < 1 { diff --git a/share/namespace_test.go b/share/namespace_test.go index 8cc61b379b..62746e4b4e 100644 --- a/share/namespace_test.go +++ b/share/namespace_test.go @@ -19,6 +19,59 @@ var ( invalidPrefixID = bytes.Repeat([]byte{1}, NamespaceSize) ) +func TestNewNamespaceV0(t *testing.T) { + type testCase struct { + name string + subNid []byte + expected Namespace + wantErr bool + } + testCases := []testCase{ + { + name: "8 byte id, gets left padded", + subNid: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, + expected: Namespace{ + 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // filled zeros + 0x0, 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8}, // id with left padding + wantErr: false, + }, + { + name: "10 byte id, no padding", + subNid: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x9, 0x10}, + expected: Namespace{ + 0x0, // version + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // filled zeros + 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0x10}, // id + wantErr: false, + }, + { + name: "11 byte id", + subNid: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x9, 0x10, 0x11}, + expected: []byte{}, + wantErr: true, + }, + { + name: "nil id", + subNid: nil, + expected: []byte{}, + wantErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := NewBlobNamespaceV0(tc.subNid) + if tc.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.expected, got) + }) + } +} + func TestFrom(t *testing.T) { type testCase struct { name string diff --git a/share/nid.go b/share/nid.go deleted file mode 100644 index 7d960cc9e1..0000000000 --- a/share/nid.go +++ /dev/null @@ -1,29 +0,0 @@ -package share - -import ( - "fmt" - - appns "github.com/celestiaorg/celestia-app/pkg/namespace" - "github.com/celestiaorg/nmt/namespace" -) - -// NewNamespaceV0 takes a variable size byte slice and creates a version 0 Namespace ID. -// The byte slice must be <= 10 bytes. -// If it is less than 10 bytes, it will be left padded to size 10 with 0s. -// TODO: Adapt for Namespace in the integration PR -func NewNamespaceV0(subNId []byte) (namespace.ID, error) { - if lnid := len(subNId); lnid > appns.NamespaceVersionZeroIDSize { - return nil, fmt.Errorf("namespace id must be <= %v, but it was %v bytes", appns.NamespaceVersionZeroIDSize, lnid) - } - - id := make([]byte, appns.NamespaceIDSize) - leftPaddingOffset := appns.NamespaceVersionZeroIDSize - len(subNId) - copy(id[appns.NamespaceVersionZeroPrefixSize+leftPaddingOffset:], subNId) - - appID, err := appns.New(appns.NamespaceVersionZero, id) - if err != nil { - return nil, err - } - - return appID.Bytes(), nil -} diff --git a/share/nid_test.go b/share/nid_test.go deleted file mode 100644 index 8f83d430e3..0000000000 --- a/share/nid_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package share - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/celestiaorg/nmt/namespace" -) - -func TestNewNamespaceV0(t *testing.T) { - type testCase struct { - name string - subNid []byte - expected namespace.ID - wantErr bool - } - testCases := []testCase{ - { - name: "8 byte subNid, gets left padded", - subNid: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, - expected: namespace.ID{ - 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // filled zeros - 0x0, 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8}, // id with left padding - wantErr: false, - }, - { - name: "10 byte subNid, no padding", - subNid: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x9, 0x10}, - expected: namespace.ID{ - 0x0, // version - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // filled zeros - 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0x10}, // id - wantErr: false, - }, - { - name: "11 byte subNid", - subNid: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x9, 0x10, 0x11}, - expected: []byte{}, - wantErr: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - got, err := NewNamespaceV0(tc.subNid) - if tc.wantErr { - assert.Error(t, err) - return - } - assert.NoError(t, err) - assert.Equal(t, tc.expected, got) - }) - } -} diff --git a/share/p2p/shrexeds/exchange_test.go b/share/p2p/shrexeds/exchange_test.go index b0e11e3587..21fe9f77a1 100644 --- a/share/p2p/shrexeds/exchange_test.go +++ b/share/p2p/shrexeds/exchange_test.go @@ -16,8 +16,8 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/p2p" ) @@ -34,7 +34,7 @@ func TestExchange_RequestEDS(t *testing.T) { // Testcase: EDS is immediately available t.Run("EDS_Available", func(t *testing.T) { - eds := share.RandEDS(t, 4) + eds := edstest.RandEDS(t, 4) dah := da.NewDataAvailabilityHeader(eds) err = store.Put(ctx, dah.Hash(), eds) require.NoError(t, err) @@ -47,7 +47,7 @@ func TestExchange_RequestEDS(t *testing.T) { // Testcase: EDS is unavailable initially, but is found after multiple requests t.Run("EDS_AvailableAfterDelay", func(t *testing.T) { storageDelay := time.Second - eds := share.RandEDS(t, 4) + eds := edstest.RandEDS(t, 4) dah := da.NewDataAvailabilityHeader(eds) go func() { time.Sleep(storageDelay) @@ -76,7 +76,7 @@ func TestExchange_RequestEDS(t *testing.T) { t.Run("EDS_err_not_found", func(t *testing.T) { timeoutCtx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) - eds := share.RandEDS(t, 4) + eds := edstest.RandEDS(t, 4) dah := da.NewDataAvailabilityHeader(eds) _, err := client.RequestEDS(timeoutCtx, dah.Hash(), server.host.ID()) require.ErrorIs(t, err, p2p.ErrNotFound) diff --git a/share/p2p/shrexnd/client.go b/share/p2p/shrexnd/client.go index b59bb347a7..7f12c21cf2 100644 --- a/share/p2p/shrexnd/client.go +++ b/share/p2p/shrexnd/client.go @@ -15,7 +15,6 @@ import ( "github.com/celestiaorg/go-libp2p-messenger/serde" "github.com/celestiaorg/nmt" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/ipld" @@ -51,10 +50,14 @@ func NewClient(params *Parameters, host host.Host) (*Client, error) { func (c *Client) RequestND( ctx context.Context, root *share.Root, - nID namespace.ID, + namespace share.Namespace, peer peer.ID, ) (share.NamespacedShares, error) { - shares, err := c.doRequest(ctx, root, nID, peer) + if err := namespace.ValidateForData(); err != nil { + return nil, err + } + + shares, err := c.doRequest(ctx, root, namespace, peer) if err == nil { return shares, nil } @@ -80,7 +83,7 @@ func (c *Client) RequestND( func (c *Client) doRequest( ctx context.Context, root *share.Root, - nID namespace.ID, + namespace share.Namespace, peerID peer.ID, ) (share.NamespacedShares, error) { stream, err := c.host.NewStream(ctx, peerID, c.protocolID) @@ -92,8 +95,8 @@ func (c *Client) doRequest( c.setStreamDeadlines(ctx, stream) req := &pb.GetSharesByNamespaceRequest{ - RootHash: root.Hash(), - NamespaceId: nID, + RootHash: root.Hash(), + Namespace: namespace, } _, err = serde.Write(stream, req) diff --git a/share/p2p/shrexnd/exchange_test.go b/share/p2p/shrexnd/exchange_test.go index 25528595d7..e8d3e439c0 100644 --- a/share/p2p/shrexnd/exchange_test.go +++ b/share/p2p/shrexnd/exchange_test.go @@ -14,13 +14,13 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/namespace" - nmtnamespace "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/p2p" + "github.com/celestiaorg/celestia-node/share/sharetest" ) func TestExchange_RequestND_NotFound(t *testing.T) { @@ -35,8 +35,8 @@ func TestExchange_RequestND_NotFound(t *testing.T) { t.Cleanup(cancel) root := share.Root{} - nID := make([]byte, namespace.NamespaceSize) - _, err := client.RequestND(ctx, &root, nID, server.host.ID()) + namespace := sharetest.RandV0Namespace() + _, err := client.RequestND(ctx, &root, namespace, server.host.ID()) require.ErrorIs(t, err, p2p.ErrNotFound) }) @@ -44,12 +44,12 @@ func TestExchange_RequestND_NotFound(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) - eds := share.RandEDS(t, 4) + eds := edstest.RandEDS(t, 4) dah := da.NewDataAvailabilityHeader(eds) require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) - randNID := dah.RowRoots[(len(dah.RowRoots)-1)/2][:namespace.NamespaceSize] - emptyShares, err := client.RequestND(ctx, &dah, randNID, server.host.ID()) + randNamespace := dah.RowRoots[(len(dah.RowRoots)-1)/2][:share.NamespaceSize] + emptyShares, err := client.RequestND(ctx, &dah, randNamespace, server.host.ID()) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) }) @@ -92,13 +92,13 @@ func TestExchange_RequestND(t *testing.T) { // take server concurrency slots with blocked requests for i := 0; i < rateLimit; i++ { go func(i int) { - client.RequestND(ctx, nil, nil, server.host.ID()) //nolint:errcheck + client.RequestND(ctx, nil, sharetest.RandV0Namespace(), server.host.ID()) //nolint:errcheck }(i) } // wait until all server slots are taken wg.Wait() - _, err = client.RequestND(ctx, nil, nil, server.host.ID()) + _, err = client.RequestND(ctx, nil, sharetest.RandV0Namespace(), server.host.ID()) require.ErrorIs(t, err, p2p.ErrNotFound) }) } @@ -118,7 +118,7 @@ func (m notFoundGetter) GetEDS( } func (m notFoundGetter) GetSharesByNamespace( - _ context.Context, _ *share.Root, _ nmtnamespace.ID, + _ context.Context, _ *share.Root, _ share.Namespace, ) (share.NamespacedShares, error) { return nil, nil } diff --git a/share/p2p/shrexnd/pb/share.pb.go b/share/p2p/shrexnd/pb/share.pb.go index 7e19bebc09..e4c7656b4e 100644 --- a/share/p2p/shrexnd/pb/share.pb.go +++ b/share/p2p/shrexnd/pb/share.pb.go @@ -57,8 +57,8 @@ func (StatusCode) EnumDescriptor() ([]byte, []int) { } type GetSharesByNamespaceRequest struct { - RootHash []byte `protobuf:"bytes,1,opt,name=root_hash,json=rootHash,proto3" json:"root_hash,omitempty"` - NamespaceId []byte `protobuf:"bytes,2,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + RootHash []byte `protobuf:"bytes,1,opt,name=root_hash,json=rootHash,proto3" json:"root_hash,omitempty"` + Namespace []byte `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"` } func (m *GetSharesByNamespaceRequest) Reset() { *m = GetSharesByNamespaceRequest{} } @@ -101,9 +101,9 @@ func (m *GetSharesByNamespaceRequest) GetRootHash() []byte { return nil } -func (m *GetSharesByNamespaceRequest) GetNamespaceId() []byte { +func (m *GetSharesByNamespaceRequest) GetNamespace() []byte { if m != nil { - return m.NamespaceId + return m.Namespace } return nil } @@ -291,33 +291,32 @@ func init() { func init() { proto.RegisterFile("share/p2p/shrexnd/pb/share.proto", fileDescriptor_ed9f13149b0de397) } var fileDescriptor_ed9f13149b0de397 = []byte{ - // 401 bytes of a gzipped FileDescriptorProto + // 396 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x92, 0xcf, 0x6e, 0xd3, 0x40, - 0x10, 0xc6, 0xe3, 0x6c, 0x62, 0xd2, 0xb1, 0x41, 0xd6, 0x80, 0xa8, 0xa1, 0xc8, 0x0a, 0x3e, 0x45, - 0x20, 0xd9, 0x92, 0x91, 0xb8, 0xbb, 0x6d, 0x00, 0x8b, 0xb2, 0xa9, 0x36, 0x05, 0x4e, 0xc8, 0xda, - 0xe2, 0xad, 0x8c, 0x04, 0xde, 0xc5, 0xbb, 0x55, 0xe0, 0xcc, 0x0b, 0xf0, 0x58, 0x1c, 0x7b, 0xe4, - 0x88, 0x92, 0x17, 0x41, 0x5e, 0xa7, 0xf4, 0x40, 0x6e, 0xfb, 0xcd, 0xfc, 0xe6, 0xdf, 0xa7, 0x85, - 0xa9, 0xae, 0x79, 0x2b, 0x52, 0x95, 0xa9, 0x54, 0xd7, 0xad, 0xf8, 0xd6, 0x54, 0xa9, 0x3a, 0x4f, - 0x6d, 0x30, 0x51, 0xad, 0x34, 0x12, 0x71, 0x2b, 0x32, 0x95, 0x58, 0x22, 0x69, 0xaa, 0xf8, 0x03, - 0x1c, 0xbc, 0x14, 0x66, 0xd9, 0x25, 0xf4, 0xe1, 0x77, 0xca, 0xbf, 0x08, 0xad, 0xf8, 0x47, 0xc1, - 0xc4, 0xd7, 0x4b, 0xa1, 0x0d, 0x1e, 0xc0, 0x5e, 0x2b, 0xa5, 0x29, 0x6b, 0xae, 0xeb, 0xd0, 0x99, - 0x3a, 0x33, 0x9f, 0x4d, 0xba, 0xc0, 0x2b, 0xae, 0x6b, 0x7c, 0x0c, 0x7e, 0x73, 0x5d, 0x50, 0x7e, - 0xaa, 0xc2, 0xa1, 0xcd, 0x7b, 0xff, 0x62, 0x45, 0x15, 0xff, 0x70, 0xe0, 0xd1, 0xee, 0xfe, 0x5a, - 0xc9, 0x46, 0x0b, 0x7c, 0x0e, 0xae, 0x36, 0xdc, 0x5c, 0x6a, 0xdb, 0xfd, 0x4e, 0x16, 0x25, 0xff, - 0x2f, 0x99, 0x2c, 0x2d, 0x71, 0x24, 0x2b, 0xc1, 0xb6, 0x34, 0x3e, 0x85, 0x51, 0x2b, 0x57, 0x3a, - 0x1c, 0x4e, 0xc9, 0xcc, 0xcb, 0xf6, 0x77, 0x55, 0x31, 0xb9, 0x62, 0x16, 0x8a, 0x29, 0x10, 0x26, - 0x57, 0x78, 0x1f, 0x5c, 0x8b, 0x75, 0xb3, 0xc8, 0xcc, 0x67, 0x5b, 0x85, 0x29, 0x8c, 0x55, 0x2b, - 0xe5, 0x85, 0x3d, 0xc0, 0xcb, 0x1e, 0xec, 0x6a, 0x76, 0xda, 0x01, 0xac, 0xe7, 0x62, 0x0e, 0x63, - 0xab, 0xf1, 0x1e, 0x8c, 0xb5, 0xe1, 0xad, 0xb1, 0xcb, 0x13, 0xd6, 0x0b, 0x0c, 0x80, 0x88, 0xa6, - 0xb7, 0x83, 0xb0, 0xee, 0xd9, 0x71, 0x8d, 0xac, 0x84, 0x0e, 0x89, 0x1d, 0xdc, 0x0b, 0x7c, 0x08, - 0x93, 0xce, 0xd7, 0xcf, 0x82, 0x5f, 0x84, 0xa3, 0xde, 0xdb, 0x6b, 0xfd, 0xe4, 0x3d, 0xc0, 0xcd, - 0xd5, 0xe8, 0xc1, 0xad, 0x82, 0xbe, 0xcb, 0x4f, 0x8a, 0xe3, 0x60, 0x80, 0x2e, 0x0c, 0x17, 0xaf, - 0x03, 0x07, 0x6f, 0xc3, 0x1e, 0x5d, 0x9c, 0x95, 0x2f, 0x16, 0x6f, 0xe9, 0x71, 0x30, 0x44, 0x1f, - 0x26, 0x05, 0x3d, 0x9b, 0x33, 0x9a, 0x9f, 0x04, 0x04, 0xf7, 0xe1, 0x2e, 0xcd, 0xdf, 0xcc, 0x97, - 0xa7, 0xf9, 0xd1, 0xbc, 0xbc, 0xc1, 0x46, 0x87, 0xe1, 0xaf, 0x75, 0xe4, 0x5c, 0xad, 0x23, 0xe7, - 0xcf, 0x3a, 0x72, 0x7e, 0x6e, 0xa2, 0xc1, 0xd5, 0x26, 0x1a, 0xfc, 0xde, 0x44, 0x83, 0x73, 0xd7, - 0xfe, 0x92, 0x67, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xd2, 0x3e, 0x78, 0x01, 0x49, 0x02, 0x00, - 0x00, + 0x10, 0xc6, 0x6d, 0x6f, 0x62, 0x92, 0x49, 0x40, 0xd6, 0x80, 0xa8, 0xa1, 0x95, 0x15, 0xf9, 0x14, + 0x81, 0x64, 0x4b, 0x46, 0xe2, 0xee, 0xb6, 0x01, 0x22, 0xca, 0xa6, 0xda, 0x94, 0x3f, 0xb7, 0x68, + 0x8b, 0xb7, 0xf2, 0x01, 0xbc, 0x8b, 0x77, 0xa3, 0xc0, 0x99, 0x17, 0xe0, 0xb1, 0x38, 0xe6, 0xc8, + 0x11, 0x25, 0x2f, 0x82, 0xbc, 0x4e, 0x94, 0x03, 0xb9, 0xf9, 0x9b, 0xf9, 0xcd, 0x37, 0x9f, 0x47, + 0x0b, 0x23, 0x5d, 0xf2, 0x5a, 0xa4, 0x2a, 0x53, 0xa9, 0x2e, 0x6b, 0xf1, 0xbd, 0x2a, 0x52, 0x75, + 0x9b, 0xda, 0x62, 0xa2, 0x6a, 0x69, 0x24, 0xe2, 0x4e, 0x64, 0x2a, 0xb1, 0x44, 0x52, 0x15, 0xf1, + 0x27, 0x38, 0x7d, 0x2d, 0xcc, 0xbc, 0x69, 0xe8, 0xf3, 0x1f, 0x94, 0x7f, 0x15, 0x5a, 0xf1, 0xcf, + 0x82, 0x89, 0x6f, 0x4b, 0xa1, 0x0d, 0x9e, 0x42, 0xbf, 0x96, 0xd2, 0x2c, 0x4a, 0xae, 0xcb, 0xd0, + 0x1d, 0xb9, 0xe3, 0x21, 0xeb, 0x35, 0x85, 0x37, 0x5c, 0x97, 0x78, 0x06, 0xfd, 0x6a, 0x3f, 0x10, + 0x7a, 0xb6, 0x79, 0x28, 0xc4, 0x3f, 0x5d, 0x38, 0x3b, 0x6e, 0xad, 0x95, 0xac, 0xb4, 0xc0, 0x97, + 0xe0, 0x6b, 0xc3, 0xcd, 0x52, 0x5b, 0xe3, 0x07, 0x59, 0x94, 0xfc, 0x9f, 0x2f, 0x99, 0x5b, 0xe2, + 0x42, 0x16, 0x82, 0xed, 0x68, 0x7c, 0x0e, 0x9d, 0x5a, 0xae, 0x74, 0xe8, 0x8d, 0xc8, 0x78, 0x90, + 0x9d, 0x1c, 0x9b, 0x62, 0x72, 0xc5, 0x2c, 0x14, 0x53, 0x20, 0x4c, 0xae, 0xf0, 0x31, 0xf8, 0x16, + 0x6b, 0x76, 0x91, 0xf1, 0x90, 0xed, 0x14, 0xa6, 0xd0, 0x55, 0xb5, 0x94, 0x77, 0x36, 0xfe, 0x20, + 0x7b, 0x72, 0xcc, 0xec, 0xba, 0x01, 0x58, 0xcb, 0xc5, 0x1c, 0xba, 0x56, 0xe3, 0x23, 0xe8, 0x6a, + 0xc3, 0x6b, 0x63, 0xc3, 0x13, 0xd6, 0x0a, 0x0c, 0x80, 0x88, 0xaa, 0xb0, 0x6e, 0x84, 0x35, 0x9f, + 0x0d, 0x57, 0xc9, 0x42, 0xe8, 0x90, 0xd8, 0xc5, 0xad, 0xc0, 0xa7, 0xd0, 0x6b, 0x4e, 0xfa, 0x45, + 0xf0, 0xbb, 0xb0, 0xd3, 0x9e, 0x75, 0xaf, 0x9f, 0x7d, 0x04, 0x38, 0xfc, 0x35, 0x0e, 0xe0, 0xde, + 0x94, 0x7e, 0xc8, 0xaf, 0xa6, 0x97, 0x81, 0x83, 0x3e, 0x78, 0xb3, 0xb7, 0x81, 0x8b, 0xf7, 0xa1, + 0x4f, 0x67, 0x37, 0x8b, 0x57, 0xb3, 0xf7, 0xf4, 0x32, 0xf0, 0x70, 0x08, 0xbd, 0x29, 0xbd, 0x99, + 0x30, 0x9a, 0x5f, 0x05, 0x04, 0x4f, 0xe0, 0x21, 0xcd, 0xdf, 0x4d, 0xe6, 0xd7, 0xf9, 0xc5, 0x64, + 0x71, 0xc0, 0x3a, 0xe7, 0xe1, 0xef, 0x4d, 0xe4, 0xae, 0x37, 0x91, 0xfb, 0x77, 0x13, 0xb9, 0xbf, + 0xb6, 0x91, 0xb3, 0xde, 0x46, 0xce, 0x9f, 0x6d, 0xe4, 0xdc, 0xfa, 0xf6, 0x81, 0xbc, 0xf8, 0x17, + 0x00, 0x00, 0xff, 0xff, 0x0b, 0x93, 0xfd, 0x1b, 0x44, 0x02, 0x00, 0x00, } func (m *GetSharesByNamespaceRequest) Marshal() (dAtA []byte, err error) { @@ -340,10 +339,10 @@ func (m *GetSharesByNamespaceRequest) MarshalToSizedBuffer(dAtA []byte) (int, er _ = i var l int _ = l - if len(m.NamespaceId) > 0 { - i -= len(m.NamespaceId) - copy(dAtA[i:], m.NamespaceId) - i = encodeVarintShare(dAtA, i, uint64(len(m.NamespaceId))) + if len(m.Namespace) > 0 { + i -= len(m.Namespace) + copy(dAtA[i:], m.Namespace) + i = encodeVarintShare(dAtA, i, uint64(len(m.Namespace))) i-- dAtA[i] = 0x12 } @@ -513,7 +512,7 @@ func (m *GetSharesByNamespaceRequest) Size() (n int) { if l > 0 { n += 1 + l + sovShare(uint64(l)) } - l = len(m.NamespaceId) + l = len(m.Namespace) if l > 0 { n += 1 + l + sovShare(uint64(l)) } @@ -653,7 +652,7 @@ func (m *GetSharesByNamespaceRequest) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field NamespaceId", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Namespace", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -680,9 +679,9 @@ func (m *GetSharesByNamespaceRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.NamespaceId = append(m.NamespaceId[:0], dAtA[iNdEx:postIndex]...) - if m.NamespaceId == nil { - m.NamespaceId = []byte{} + m.Namespace = append(m.Namespace[:0], dAtA[iNdEx:postIndex]...) + if m.Namespace == nil { + m.Namespace = []byte{} } iNdEx = postIndex default: diff --git a/share/p2p/shrexnd/pb/share.proto b/share/p2p/shrexnd/pb/share.proto index 09d77d29f3..306865d17f 100644 --- a/share/p2p/shrexnd/pb/share.proto +++ b/share/p2p/shrexnd/pb/share.proto @@ -4,7 +4,7 @@ package share.p2p.shrex.nd; message GetSharesByNamespaceRequest{ bytes root_hash = 1; - bytes namespace_id = 2; + bytes namespace = 2; } message GetSharesByNamespaceResponse{ diff --git a/share/p2p/shrexnd/server.go b/share/p2p/shrexnd/server.go index 72c5231eff..d61e6b2e68 100644 --- a/share/p2p/shrexnd/server.go +++ b/share/p2p/shrexnd/server.go @@ -17,7 +17,6 @@ import ( "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/p2p" pb "github.com/celestiaorg/celestia-node/share/p2p/shrexnd/pb" ) @@ -100,7 +99,7 @@ func (srv *Server) handleNamespacedData(ctx context.Context, stream network.Stre stream.Reset() //nolint:errcheck return } - logger = logger.With("namespaceId", hex.EncodeToString(req.NamespaceId), "hash", share.DataHash(req.RootHash).String()) + logger = logger.With("namespace", hex.EncodeToString(req.Namespace), "hash", share.DataHash(req.RootHash).String()) logger.Debugw("server: new request") err = stream.CloseRead() @@ -130,7 +129,7 @@ func (srv *Server) handleNamespacedData(ctx context.Context, stream network.Stre return } - shares, err := srv.getter.GetSharesByNamespace(ctx, dah, req.NamespaceId) + shares, err := srv.getter.GetSharesByNamespace(ctx, dah, req.Namespace) switch { case errors.Is(err, share.ErrNotFound): logger.Warn("server: nd not found") @@ -148,13 +147,12 @@ func (srv *Server) handleNamespacedData(ctx context.Context, stream network.Stre // validateRequest checks correctness of the request func validateRequest(req pb.GetSharesByNamespaceRequest) error { - if len(req.NamespaceId) != ipld.NamespaceSize { - return fmt.Errorf("incorrect namespace id length: %v", len(req.NamespaceId)) + if err := share.Namespace(req.Namespace).ValidateForData(); err != nil { + return err } if len(req.RootHash) != sha256.Size { return fmt.Errorf("incorrect root hash length: %v", len(req.RootHash)) } - return nil } diff --git a/share/share.go b/share/share.go index 0178054a9f..02ccd73909 100644 --- a/share/share.go +++ b/share/share.go @@ -4,32 +4,23 @@ import ( "bytes" "fmt" - "go.opentelemetry.io/otel" - "github.com/celestiaorg/celestia-app/pkg/appconsts" - "github.com/celestiaorg/nmt/namespace" - - "github.com/celestiaorg/celestia-node/share/ipld" ) var ( - tracer = otel.Tracer("share") - // DefaultRSMT2DCodec sets the default rsmt2d.Codec for shares. DefaultRSMT2DCodec = appconsts.DefaultCodec ) const ( - // NamespaceSize is a system-wide size for NMT namespaces. - NamespaceSize = appconsts.NamespaceSize - // Size is a system-wide size of a share, including both data and namespace ID + // Size is a system-wide size of a share, including both data and namespace GetNamespace Size = appconsts.ShareSize ) var ( // MaxSquareSize is currently the maximum size supported for unerasured data in // rsmt2d.ExtendedDataSquare. - MaxSquareSize = ipld.MaxSquareSize + MaxSquareSize = appconsts.SquareSizeUpperBound(appconsts.LatestVersion) ) // Share contains the raw share data without the corresponding namespace. @@ -38,13 +29,13 @@ var ( // on it. type Share = []byte -// ID gets the namespace ID from the share. -func ID(s Share) namespace.ID { +// GetNamespace slices Namespace out of the Share. +func GetNamespace(s Share) Namespace { return s[:NamespaceSize] } -// Data gets data from the share. -func Data(s Share) []byte { +// GetData slices out data of the Share. +func GetData(s Share) []byte { return s[NamespaceSize:] } diff --git a/share/sharetest/testing.go b/share/sharetest/testing.go new file mode 100644 index 0000000000..50eb0eb003 --- /dev/null +++ b/share/sharetest/testing.go @@ -0,0 +1,58 @@ +package sharetest + +import ( + "bytes" + "math/rand" + "sort" + "sync" + "time" + + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-app/pkg/namespace" + + "github.com/celestiaorg/celestia-node/share" +) + +// RandShares generate 'total' amount of shares filled with random data. It uses require.TestingT +// to be able to take both a *testing.T and a *testing.B. +func RandShares(t require.TestingT, total int) []share.Share { + if total&(total-1) != 0 { + t.Errorf("total must be power of 2: %d", total) + t.FailNow() + } + + shares := make([]share.Share, total) + for i := range shares { + shr := make([]byte, share.Size) + copy(share.GetNamespace(shr), RandV0Namespace()) + rndMu.Lock() + _, err := rnd.Read(share.GetData(shr)) + rndMu.Unlock() + require.NoError(t, err) + shares[i] = shr + } + sort.Slice(shares, func(i, j int) bool { return bytes.Compare(shares[i], shares[j]) < 0 }) + + return shares +} + +// RandV0Namespace generates random valid data namespace for testing purposes. +func RandV0Namespace() share.Namespace { + rb := make([]byte, namespace.NamespaceVersionZeroIDSize) + rndMu.Lock() + rnd.Read(rb) + rndMu.Unlock() + for { + namespace, _ := share.NewBlobNamespaceV0(rb) + if err := namespace.ValidateForData(); err != nil { + continue + } + return namespace + } +} + +var ( + rnd = rand.New(rand.NewSource(time.Now().Unix())) //nolint:gosec + rndMu sync.Mutex +) diff --git a/share/test_helpers.go b/share/test_helpers.go deleted file mode 100644 index c02bfc55ac..0000000000 --- a/share/test_helpers.go +++ /dev/null @@ -1,65 +0,0 @@ -package share - -import ( - "bytes" - "crypto/rand" - "sort" - - "github.com/stretchr/testify/require" - - "github.com/celestiaorg/celestia-app/pkg/namespace" - "github.com/celestiaorg/celestia-app/pkg/wrapper" - "github.com/celestiaorg/rsmt2d" -) - -// EqualEDS check whether two given EDSes are equal. -// TODO(Wondertan): Move to rsmt2d -// TODO(Wondertan): Propose use of int by default instead of uint for the sake convenience and -// Golang practices -func EqualEDS(a *rsmt2d.ExtendedDataSquare, b *rsmt2d.ExtendedDataSquare) bool { - if a.Width() != b.Width() { - return false - } - - for i := uint(0); i < a.Width(); i++ { - ar, br := a.Row(i), b.Row(i) - for j := 0; j < len(ar); j++ { - if !bytes.Equal(ar[j], br[j]) { - return false - } - } - } - - return true -} - -// RandEDS generates EDS filled with the random data with the given size for original square. It -// uses require.TestingT to be able to take both a *testing.T and a *testing.B. -func RandEDS(t require.TestingT, size int) *rsmt2d.ExtendedDataSquare { - shares := RandShares(t, size*size) - // recompute the eds - eds, err := rsmt2d.ComputeExtendedDataSquare(shares, DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(size))) - require.NoError(t, err, "failure to recompute the extended data square") - return eds -} - -// RandShares generate 'total' amount of shares filled with random data. It uses require.TestingT -// to be able to take both a *testing.T and a *testing.B. -func RandShares(t require.TestingT, total int) []Share { - if total&(total-1) != 0 { - t.Errorf("total must be power of 2: %d", total) - t.FailNow() - } - - shares := make([]Share, total) - for i := range shares { - share := make([]byte, Size) - copy(share[:NamespaceSize], namespace.RandomNamespace().Bytes()) - _, err := rand.Read(share[NamespaceSize:]) - require.NoError(t, err) - shares[i] = share - } - sort.Slice(shares, func(i, j int) bool { return bytes.Compare(shares[i], shares[j]) < 0 }) - - return shares -} diff --git a/state/core_access.go b/state/core_access.go index 7b59f3e714..3fefaa3ed9 100644 --- a/state/core_access.go +++ b/state/core_access.go @@ -167,8 +167,11 @@ func (ca *CoreAccessor) SubmitPayForBlob( } appblobs := make([]*apptypes.Blob, len(blobs)) - for i, blob := range blobs { - appblobs[i] = &blob.Blob + for i, b := range blobs { + if err := b.Namespace().ValidateForBlob(); err != nil { + return nil, err + } + appblobs[i] = &b.Blob } response, err := appblob.SubmitPayForBlob(