From 4a2ef55f141ccbdb4f9049d84356907312e69a35 Mon Sep 17 00:00:00 2001 From: Wondertan Date: Tue, 22 Aug 2023 16:26:47 +0200 Subject: [PATCH 1/2] chore: migrate to boxo --- Makefile | 5 +++- blob/service_test.go | 7 ++--- das/daser_test.go | 10 +++---- go.mod | 11 +++---- go.sum | 15 +++++----- header/headertest/fraud/testing.go | 5 ++-- nodebuilder/node.go | 2 +- nodebuilder/p2p/ipld.go | 12 -------- nodebuilder/p2p/module.go | 3 +- nodebuilder/share/constructors.go | 2 +- nodebuilder/tests/fraud_test.go | 2 +- share/availability/cache/availability_test.go | 4 +-- share/availability/cache/testing.go | 6 ++-- share/availability/full/testing.go | 4 +-- share/availability/light/testing.go | 8 ++--- share/availability/test/testing.go | 4 +-- share/eds/adapters.go | 2 +- share/eds/byzantine/bad_encoding_test.go | 5 ++-- share/eds/byzantine/byzantine.go | 2 +- share/eds/byzantine/share_proof.go | 2 +- share/eds/byzantine/share_proof_test.go | 5 ++-- share/eds/retriever.go | 2 +- share/eds/retriever_test.go | 15 +++++----- share/getters/getter_test.go | 6 ++-- share/getters/ipld.go | 2 +- share/getters/utils.go | 2 +- share/ipld/add.go | 5 ++-- share/ipld/blockserv.go | 30 +++++++++++++++++++ share/ipld/get.go | 2 +- share/ipld/get_shares.go | 2 +- share/ipld/get_shares_test.go | 25 +++++++--------- share/ipld/namespace_data.go | 2 +- share/ipld/nmt.go | 2 +- share/ipld/nmt_adder.go | 4 +-- 34 files changed, 112 insertions(+), 103 deletions(-) delete mode 100644 nodebuilder/p2p/ipld.go create mode 100644 share/ipld/blockserv.go diff --git a/Makefile b/Makefile index d4b3d2572e..8813b65d23 100644 --- a/Makefile +++ b/Makefile @@ -169,7 +169,10 @@ lint-imports: ## sort-imports: Sort Go imports. sort-imports: - @goimports-reviser -company-prefixes "github.com/celestiaorg" -project-name "github.com/celestiaorg/"$(PROJECTNAME)"" -output stdout . + @for file in `find . -type f -name '*.go'`; \ + do goimports-reviser -company-prefixes "github.com/celestiaorg" -project-name "github.com/celestiaorg/"$(PROJECTNAME)"" $$file \ + || exit 1; \ + done; .PHONY: sort-imports ## adr-gen: Generate ADR from template. Must set NUM and TITLE parameters. diff --git a/blob/service_test.go b/blob/service_test.go index a45fe631af..64296b969c 100644 --- a/blob/service_test.go +++ b/blob/service_test.go @@ -10,7 +10,6 @@ import ( ds "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" - mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" tmrand "github.com/tendermint/tendermint/libs/rand" @@ -324,7 +323,7 @@ func TestService_GetSingleBlobWithoutPadding(t *testing.T) { rawShares = append(rawShares, append(rawShares0, padding0.ToBytes())...) rawShares = append(rawShares, append(rawShares1, padding1.ToBytes())...) - bs := mdutils.Bserv() + bs := ipld.NewMemBlockservice() batching := ds_sync.MutexWrap(ds.NewMapDatastore()) headerStore, err := store.NewStore[*header.ExtendedHeader](batching) require.NoError(t, err) @@ -375,7 +374,7 @@ func TestService_GetAllWithoutPadding(t *testing.T) { rawShares = append(rawShares, append(rawShares0, padding0.ToBytes())...) } - bs := mdutils.Bserv() + bs := ipld.NewMemBlockservice() batching := ds_sync.MutexWrap(ds.NewMapDatastore()) headerStore, err := store.NewStore[*header.ExtendedHeader](batching) require.NoError(t, err) @@ -397,7 +396,7 @@ func TestService_GetAllWithoutPadding(t *testing.T) { } func createService(ctx context.Context, t *testing.T, blobs []*Blob) *Service { - bs := mdutils.Bserv() + bs := ipld.NewMemBlockservice() batching := ds_sync.MutexWrap(ds.NewMapDatastore()) headerStore, err := store.NewStore[*header.ExtendedHeader](batching) require.NoError(t, err) diff --git a/das/daser_test.go b/das/daser_test.go index e4e74dc7ff..3e5bce0b97 100644 --- a/das/daser_test.go +++ b/das/daser_test.go @@ -6,10 +6,9 @@ import ( "time" "github.com/golang/mock/gomock" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" - mdutils "github.com/ipfs/go-merkledag/test" pubsub "github.com/libp2p/go-libp2p-pubsub" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/assert" @@ -31,6 +30,7 @@ import ( availability_test "github.com/celestiaorg/celestia-node/share/availability/test" "github.com/celestiaorg/celestia-node/share/eds/byzantine" "github.com/celestiaorg/celestia-node/share/getters" + "github.com/celestiaorg/celestia-node/share/ipld" ) var timeout = time.Second * 15 @@ -39,7 +39,7 @@ var timeout = time.Second * 15 // the DASer checkpoint is updated to network head. func TestDASerLifecycle(t *testing.T) { ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() avail := light.TestAvailability(getters.NewIPLDGetter(bServ)) // 15 headers from the past and 15 future headers mockGet, sub, mockService := createDASerSubcomponents(t, bServ, 15, 15) @@ -79,7 +79,7 @@ func TestDASerLifecycle(t *testing.T) { func TestDASer_Restart(t *testing.T) { ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() avail := light.TestAvailability(getters.NewIPLDGetter(bServ)) // 15 headers from the past and 15 future headers mockGet, sub, mockService := createDASerSubcomponents(t, bServ, 15, 15) @@ -147,7 +147,7 @@ func TestDASer_stopsAfter_BEFP(t *testing.T) { t.Cleanup(cancel) ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() // create mock network net, err := mocknet.FullMeshLinked(1) require.NoError(t, err) diff --git a/go.mod b/go.mod index 8c1fbb1349..2306eae8b9 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,6 @@ go 1.21 toolchain go1.21.0 -replace github.com/ipfs/go-verifcid => github.com/celestiaorg/go-verifcid v0.0.1-lazypatch - require ( cosmossdk.io/errors v1.0.0 cosmossdk.io/math v1.1.2 @@ -32,15 +30,13 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.4 github.com/hashicorp/golang-lru v1.0.2 github.com/imdario/mergo v0.3.16 - github.com/ipfs/boxo v0.11.0 - github.com/ipfs/go-block-format v0.2.0 - github.com/ipfs/go-blockservice v0.5.1 // down 1 version, 0.5.2 is marked as deprecated and raises alerts + github.com/ipfs/boxo v0.12.0 + github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ipld-cbor v0.1.0 github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 - github.com/ipfs/go-merkledag v0.11.0 github.com/ipld/go-car v0.6.2 github.com/libp2p/go-libp2p v0.30.0 github.com/libp2p/go-libp2p-kad-dht v0.25.0 @@ -200,15 +196,16 @@ require ( github.com/influxdata/influxdb-client-go/v2 v2.12.2 // indirect github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect github.com/ipfs/bbloom v0.0.4 // indirect + github.com/ipfs/go-blockservice v0.5.0 github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 github.com/ipfs/go-ipfs-exchange-interface v0.2.1 // indirect - github.com/ipfs/go-ipfs-exchange-offline v0.3.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.3 // indirect github.com/ipfs/go-ipld-legacy v0.2.1 // indirect github.com/ipfs/go-log v1.0.5 // indirect + github.com/ipfs/go-merkledag v0.11.0 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/ipfs/go-peertaskqueue v0.8.1 // indirect github.com/ipfs/go-verifcid v0.0.2 // indirect diff --git a/go.sum b/go.sum index d738f67e10..c455d1def9 100644 --- a/go.sum +++ b/go.sum @@ -372,8 +372,6 @@ github.com/celestiaorg/go-header v0.3.0 h1:9fhxSgldPiWWq3yd9u7oSk5vYqaLV1JkeTnJd github.com/celestiaorg/go-header v0.3.0/go.mod h1:H8xhnDLDLbkpwmWPhCaZyTnIV3dlVxBHPnxNXS2Qu6c= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= -github.com/celestiaorg/go-verifcid v0.0.1-lazypatch h1:9TSe3w1cmJmbWlweCwCTIZkan7jV8M+KwglXpdD+UG8= -github.com/celestiaorg/go-verifcid v0.0.1-lazypatch/go.mod h1:kXPYu0XqTNUKWA1h3M95UHjUqBzDwXVVt/RXZDjKJmQ= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= github.com/celestiaorg/nmt v0.19.0 h1:9VXFeI/gt+q8h5HeCE0RjXJhOxsFzxJUjHrkvF9CMYE= @@ -1023,8 +1021,8 @@ github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1: github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.11.0 h1:urMxhZ3xoF4HssJVD3+0ssGT9pptEfHfbL8DYdoWFlg= -github.com/ipfs/boxo v0.11.0/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= +github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ= +github.com/ipfs/boxo v0.12.0/go.mod h1:xAnfiU6PtxWCnRqu7dcXQ10bB5/kvI1kXRotuGqGBhg= github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= @@ -1040,16 +1038,14 @@ github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/ github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= github.com/ipfs/go-block-format v0.1.1/go.mod h1:+McEIT+g52p+zz5xGAABGSOKrzmrdX97bc0USBdWPUs= +github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE= -github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= -github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.4/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU= github.com/ipfs/go-blockservice v0.2.1/go.mod h1:k6SiwmgyYgs4M/qt+ww6amPeUH9EISLRBnvUurKJhi8= github.com/ipfs/go-blockservice v0.3.0/go.mod h1:P5ppi8IHDC7O+pA0AlGTF09jruB2h+oP3wVVaZl8sfk= +github.com/ipfs/go-blockservice v0.5.0 h1:B2mwhhhVQl2ntW2EIpaWPwSCxSuqr5fFA93Ms4bYLEY= github.com/ipfs/go-blockservice v0.5.0/go.mod h1:W6brZ5k20AehbmERplmERn8o2Ni3ZZubvAxaIUeaT6w= -github.com/ipfs/go-blockservice v0.5.1 h1:9pAtkyKAz/skdHTh0kH8VulzWp+qmSDD0aI17TYP/s0= -github.com/ipfs/go-blockservice v0.5.1/go.mod h1:VpMblFEqG67A/H2sHKAemeH9vlURVavlysbdUI632yk= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= @@ -1219,6 +1215,9 @@ github.com/ipfs/go-unixfsnode v1.5.2/go.mod h1:NlOebRwYx8lMCNMdhAhEspYPBD3obp7TE github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipfs/go-unixfsnode v1.7.4 h1:iLvKyAVKUYOIAW2t4kDYqsT7VLGj31eXJE2aeqGfbwA= github.com/ipfs/go-unixfsnode v1.7.4/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= +github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= +github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= +github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= github.com/ipfs/interface-go-ipfs-core v0.9.0/go.mod h1:F3EcmDy53GFkF0H3iEJpfJC320fZ/4G60eftnItrrJ0= github.com/ipfs/interface-go-ipfs-core v0.10.0/go.mod h1:F3EcmDy53GFkF0H3iEJpfJC320fZ/4G60eftnItrrJ0= github.com/ipld/go-car v0.5.0/go.mod h1:ppiN5GWpjOZU9PgpAZ9HbZd9ZgSpwPMr48fGRJOWmvE= diff --git a/header/headertest/fraud/testing.go b/header/headertest/fraud/testing.go index 6a5cda733d..6e849b0ec6 100644 --- a/header/headertest/fraud/testing.go +++ b/header/headertest/fraud/testing.go @@ -2,10 +2,10 @@ package headerfraud import ( "context" + "github.com/ipfs/boxo/blockservice" "testing" "time" - "github.com/ipfs/go-blockservice" "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/libs/bytes" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" @@ -46,7 +46,8 @@ func NewFraudMaker(t *testing.T, height int64, vals []types.PrivValidator, valSe } func (f *FraudMaker) MakeExtendedHeader(odsSize int, edsStore *eds.Store) header.ConstructFn { - return func(h *types.Header, + return func( + h *types.Header, comm *types.Commit, vals *types.ValidatorSet, eds *rsmt2d.ExtendedDataSquare, diff --git a/nodebuilder/node.go b/nodebuilder/node.go index 6d83e6c4c3..58a182276c 100644 --- a/nodebuilder/node.go +++ b/nodebuilder/node.go @@ -6,8 +6,8 @@ import ( "fmt" "strings" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/boxo/exchange" - "github.com/ipfs/go-blockservice" logging "github.com/ipfs/go-log/v2" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/host" diff --git a/nodebuilder/p2p/ipld.go b/nodebuilder/p2p/ipld.go deleted file mode 100644 index ad32cc39f5..0000000000 --- a/nodebuilder/p2p/ipld.go +++ /dev/null @@ -1,12 +0,0 @@ -package p2p - -import ( - "github.com/ipfs/boxo/blockstore" - "github.com/ipfs/boxo/exchange" - "github.com/ipfs/go-blockservice" -) - -// blockService constructs IPFS's BlockService for fetching arbitrary Merkle structures. -func blockService(bs blockstore.Blockstore, ex exchange.Interface) blockservice.BlockService { - return blockservice.New(bs, ex) -} diff --git a/nodebuilder/p2p/module.go b/nodebuilder/p2p/module.go index e7f60956c7..1ddff02173 100644 --- a/nodebuilder/p2p/module.go +++ b/nodebuilder/p2p/module.go @@ -6,6 +6,7 @@ import ( "go.uber.org/fx" "github.com/celestiaorg/celestia-node/nodebuilder/node" + "github.com/celestiaorg/celestia-node/share/ipld" ) var log = logging.Logger("module/p2p") @@ -27,7 +28,7 @@ func ConstructModule(tp node.Type, cfg *Config) fx.Option { fx.Provide(routedHost), fx.Provide(pubSub), fx.Provide(dataExchange), - fx.Provide(blockService), + fx.Provide(ipld.NewBlockservice), fx.Provide(peerRouting), fx.Provide(contentRouting), fx.Provide(addrsFactory(cfg.AnnounceAddresses, cfg.NoAnnounceAddresses)), diff --git a/nodebuilder/share/constructors.go b/nodebuilder/share/constructors.go index a1b7e39713..ab8afe2fd4 100644 --- a/nodebuilder/share/constructors.go +++ b/nodebuilder/share/constructors.go @@ -5,7 +5,7 @@ import ( "errors" "github.com/filecoin-project/dagstore" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/routing" diff --git a/nodebuilder/tests/fraud_test.go b/nodebuilder/tests/fraud_test.go index 95c702c0c0..a63d934588 100644 --- a/nodebuilder/tests/fraud_test.go +++ b/nodebuilder/tests/fraud_test.go @@ -2,10 +2,10 @@ package tests import ( "context" + "github.com/ipfs/go-datastore" "testing" "time" - "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" diff --git a/share/availability/cache/availability_test.go b/share/availability/cache/availability_test.go index 03df47f848..ccc11a1d28 100644 --- a/share/availability/cache/availability_test.go +++ b/share/availability/cache/availability_test.go @@ -9,7 +9,6 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/sync" - mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -17,6 +16,7 @@ import ( "github.com/celestiaorg/celestia-node/share" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" + "github.com/celestiaorg/celestia-node/share/ipld" ) // TestCacheAvailability tests to ensure that the successful result of a @@ -86,7 +86,7 @@ func TestCacheAvailability_NoDuplicateSampling(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() // create root to cache - root := availability_test.RandFillBS(t, 16, mdutils.Bserv()) + root := availability_test.RandFillBS(t, 16, ipld.NewMemBlockservice()) // wrap dummyAvailability with a datastore ds := sync.MutexWrap(datastore.NewMapDatastore()) ca := NewShareAvailability(&dummyAvailability{counter: 0}, ds) diff --git a/share/availability/cache/testing.go b/share/availability/cache/testing.go index 978d51b6b5..00a1520114 100644 --- a/share/availability/cache/testing.go +++ b/share/availability/cache/testing.go @@ -5,18 +5,18 @@ import ( ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" - mdutils "github.com/ipfs/go-merkledag/test" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/availability/full" "github.com/celestiaorg/celestia-node/share/availability/light" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" "github.com/celestiaorg/celestia-node/share/getters" + "github.com/celestiaorg/celestia-node/share/ipld" ) // LightAvailabilityWithLocalRandSquare wraps light.GetterWithRandSquare with cache availability func LightAvailabilityWithLocalRandSquare(t *testing.T, n int) (share.Availability, *share.Root) { - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() store := dssync.MutexWrap(ds.NewMapDatastore()) getter := getters.NewIPLDGetter(bServ) avail := NewShareAvailability( @@ -28,7 +28,7 @@ func LightAvailabilityWithLocalRandSquare(t *testing.T, n int) (share.Availabili // FullAvailabilityWithLocalRandSquare wraps full.GetterWithRandSquare with cache availability func FullAvailabilityWithLocalRandSquare(t *testing.T, n int) (share.Availability, *share.Root) { - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() store := dssync.MutexWrap(ds.NewMapDatastore()) getter := getters.NewIPLDGetter(bServ) avail := NewShareAvailability( diff --git a/share/availability/full/testing.go b/share/availability/full/testing.go index 680eb7abaf..4638fde1bc 100644 --- a/share/availability/full/testing.go +++ b/share/availability/full/testing.go @@ -4,20 +4,20 @@ import ( "testing" "time" - mdutils "github.com/ipfs/go-merkledag/test" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/p2p/discovery/routing" "github.com/celestiaorg/celestia-node/share" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" "github.com/celestiaorg/celestia-node/share/getters" + "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/p2p/discovery" ) // GetterWithRandSquare provides a share.Getter filled with 'n' NMT // trees of 'n' random shares, essentially storing a whole square. func GetterWithRandSquare(t *testing.T, n int) (share.Getter, *share.Root) { - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() getter := getters.NewIPLDGetter(bServ) return getter, availability_test.RandFillBS(t, n, bServ) } diff --git a/share/availability/light/testing.go b/share/availability/light/testing.go index 59163d6356..991d143a62 100644 --- a/share/availability/light/testing.go +++ b/share/availability/light/testing.go @@ -3,18 +3,18 @@ package light import ( "testing" - "github.com/ipfs/go-blockservice" - mdutils "github.com/ipfs/go-merkledag/test" + "github.com/ipfs/boxo/blockservice" "github.com/celestiaorg/celestia-node/share" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" "github.com/celestiaorg/celestia-node/share/getters" + "github.com/celestiaorg/celestia-node/share/ipld" ) // GetterWithRandSquare provides a share.Getter filled with 'n' NMT trees of 'n' random shares, // essentially storing a whole square. func GetterWithRandSquare(t *testing.T, n int) (share.Getter, *share.Root) { - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() getter := getters.NewIPLDGetter(bServ) return getter, availability_test.RandFillBS(t, n, bServ) } @@ -22,7 +22,7 @@ func GetterWithRandSquare(t *testing.T, n int) (share.Getter, *share.Root) { // EmptyGetter provides an unfilled share.Getter with corresponding blockservice.BlockService than // can be filled by the test. func EmptyGetter() (share.Getter, blockservice.BlockService) { - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() getter := getters.NewIPLDGetter(bServ) return getter, bServ } diff --git a/share/availability/test/testing.go b/share/availability/test/testing.go index 8977678198..8973ee8cb1 100644 --- a/share/availability/test/testing.go +++ b/share/availability/test/testing.go @@ -6,9 +6,9 @@ import ( "github.com/ipfs/boxo/bitswap" "github.com/ipfs/boxo/bitswap/network" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/boxo/blockstore" "github.com/ipfs/boxo/routing/offline" - "github.com/ipfs/go-blockservice" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" record "github.com/libp2p/go-libp2p-record" @@ -94,7 +94,7 @@ func (dn *TestDagNet) NewTestNodeWithBlockstore(dstore ds.Datastore, bstore bloc ) nd := &TestNode{ net: dn, - BlockService: blockservice.New(bstore, bs), + BlockService: ipld.NewBlockservice(bstore, bs), Host: hst, } dn.nodes = append(dn.nodes, nd) diff --git a/share/eds/adapters.go b/share/eds/adapters.go index fe498400b3..8bf2340d91 100644 --- a/share/eds/adapters.go +++ b/share/eds/adapters.go @@ -5,8 +5,8 @@ import ( "sync" "github.com/filecoin-project/dagstore" + "github.com/ipfs/boxo/blockservice" blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" ) diff --git a/share/eds/byzantine/bad_encoding_test.go b/share/eds/byzantine/bad_encoding_test.go index b5dcea3452..9d16955ee3 100644 --- a/share/eds/byzantine/bad_encoding_test.go +++ b/share/eds/byzantine/bad_encoding_test.go @@ -5,7 +5,6 @@ import ( "testing" "time" - mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" core "github.com/tendermint/tendermint/types" @@ -22,7 +21,7 @@ import ( func TestBadEncodingFraudProof(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer t.Cleanup(cancel) - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() square := edstest.RandByzantineEDS(t, 16) dah, err := da.NewDataAvailabilityHeader(square) @@ -48,7 +47,7 @@ func TestIncorrectBadEncodingFraudProof(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() squareSize := 8 shares := sharetest.RandShares(t, squareSize*squareSize) diff --git a/share/eds/byzantine/byzantine.go b/share/eds/byzantine/byzantine.go index 0fcd78273e..dfdf681f04 100644 --- a/share/eds/byzantine/byzantine.go +++ b/share/eds/byzantine/byzantine.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "golang.org/x/sync/errgroup" "github.com/celestiaorg/celestia-app/pkg/da" diff --git a/share/eds/byzantine/share_proof.go b/share/eds/byzantine/share_proof.go index 411fd85818..98b58ebbec 100644 --- a/share/eds/byzantine/share_proof.go +++ b/share/eds/byzantine/share_proof.go @@ -4,7 +4,7 @@ import ( "context" "crypto/sha256" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" diff --git a/share/eds/byzantine/share_proof_test.go b/share/eds/byzantine/share_proof_test.go index db1db64f80..a9021d806d 100644 --- a/share/eds/byzantine/share_proof_test.go +++ b/share/eds/byzantine/share_proof_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/ipfs/go-cid" - mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" @@ -21,7 +20,7 @@ func TestGetProof(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) defer cancel() - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() shares := sharetest.RandShares(t, width*width) in, err := ipld.AddShares(ctx, shares, bServ) @@ -58,7 +57,7 @@ func TestGetProofs(t *testing.T) { const width = 4 ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) defer cancel() - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() shares := sharetest.RandShares(t, width*width) in, err := ipld.AddShares(ctx, shares, bServ) diff --git a/share/eds/retriever.go b/share/eds/retriever.go index e7837ae23a..c2966c3953 100644 --- a/share/eds/retriever.go +++ b/share/eds/retriever.go @@ -7,7 +7,7 @@ import ( "sync/atomic" "time" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "go.opentelemetry.io/otel" diff --git a/share/eds/retriever_test.go b/share/eds/retriever_test.go index 12b1c11083..f3f7ccca64 100644 --- a/share/eds/retriever_test.go +++ b/share/eds/retriever_test.go @@ -7,8 +7,7 @@ import ( "testing" "time" - "github.com/ipfs/go-blockservice" - mdutils "github.com/ipfs/go-merkledag/test" + "github.com/ipfs/boxo/blockservice" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -30,7 +29,7 @@ func TestRetriever_Retrieve(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() r := NewRetriever(bServ) type test struct { @@ -73,7 +72,7 @@ func TestRetriever_ByzantineError(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - bserv := mdutils.Bserv() + bserv := ipld.NewMemBlockservice() shares := edstest.RandEDS(t, width).Flattened() _, err := ipld.ImportShares(ctx, shares, bserv) require.NoError(t, err) @@ -110,7 +109,7 @@ func TestRetriever_MultipleRandQuadrants(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() r := NewRetriever(bServ) // generate EDS @@ -136,7 +135,7 @@ func TestRetriever_MultipleRandQuadrants(t *testing.T) { func TestFraudProofValidation(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer t.Cleanup(cancel) - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() odsSize := []int{2, 4, 16, 32, 64, 128} for _, size := range odsSize { @@ -178,7 +177,7 @@ BenchmarkBEFPValidation/ods_size:128 259 4934375 ns/op 5418406 B func BenchmarkBEFPValidation(b *testing.B) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer b.Cleanup(cancel) - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() r := NewRetriever(bServ) t := &testing.T{} odsSize := []int{2, 4, 16, 32, 64, 128} @@ -217,7 +216,7 @@ func BenchmarkNewErrByzantineData(b *testing.B) { odsSize := []int{2, 4, 16, 32, 64, 128} ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() r := NewRetriever(bServ) t := &testing.T{} for _, size := range odsSize { diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index 02e075459b..b0fc7e551d 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -6,10 +6,8 @@ import ( "time" "github.com/ipfs/boxo/exchange/offline" - bsrv "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" - mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -36,7 +34,7 @@ func TestTeeGetter(t *testing.T) { err = edsStore.Start(ctx) require.NoError(t, err) - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() ig := NewIPLDGetter(bServ) tg := NewTeeGetter(ig, edsStore) @@ -168,7 +166,7 @@ func TestIPLDGetter(t *testing.T) { require.NoError(t, err) bStore := edsStore.Blockstore() - bserv := bsrv.New(bStore, offline.Exchange(bStore)) + bserv := ipld.NewBlockservice(bStore, offline.Exchange(edsStore.Blockstore())) sg := NewIPLDGetter(bserv) t.Run("GetShare", func(t *testing.T) { diff --git a/share/getters/ipld.go b/share/getters/ipld.go index 8e11a389bd..4024a67f5b 100644 --- a/share/getters/ipld.go +++ b/share/getters/ipld.go @@ -7,7 +7,7 @@ import ( "sync" "sync/atomic" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/share/getters/utils.go b/share/getters/utils.go index 5305c5c737..e3f24b3857 100644 --- a/share/getters/utils.go +++ b/share/getters/utils.go @@ -6,7 +6,7 @@ import ( "fmt" "time" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "go.opentelemetry.io/otel" diff --git a/share/ipld/add.go b/share/ipld/add.go index 7e5909669d..fbb743148c 100644 --- a/share/ipld/add.go +++ b/share/ipld/add.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/nmt" @@ -52,7 +52,8 @@ func AddShares( func ImportShares( ctx context.Context, shares [][]byte, - adder blockservice.BlockService) (*rsmt2d.ExtendedDataSquare, error) { + adder blockservice.BlockService, +) (*rsmt2d.ExtendedDataSquare, error) { if len(shares) == 0 { return nil, fmt.Errorf("ipld: importing empty data") } diff --git a/share/ipld/blockserv.go b/share/ipld/blockserv.go new file mode 100644 index 0000000000..2ed2a21c77 --- /dev/null +++ b/share/ipld/blockserv.go @@ -0,0 +1,30 @@ +package ipld + +import ( + "github.com/ipfs/boxo/blockservice" + "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/exchange" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/sync" +) + +// NewBlockservice constructs Blockservice for fetching NMTrees. +func NewBlockservice(bs blockstore.Blockstore, exchange exchange.Interface) blockservice.BlockService { + return blockservice.New(bs, exchange, blockservice.WithAllowlist(defaultAllowlist)) +} + +// NewMemBlockservice constructs Blockservice for fetching NMTrees with in-memory blockstore. +func NewMemBlockservice() blockservice.BlockService { + bstore := blockstore.NewBlockstore(sync.MutexWrap(datastore.NewMapDatastore())) + return NewBlockservice(bstore, nil) +} + +// defaultAllowlist keeps default list of hashes allowed in the network. +var defaultAllowlist allowlist + +type allowlist struct{} + +func (a allowlist) IsAllowed(code uint64) bool { + // we allow all codes except home-baked sha256NamespaceFlagged + return code == sha256NamespaceFlagged +} diff --git a/share/ipld/get.go b/share/ipld/get.go index 35f601853d..f263877dc0 100644 --- a/share/ipld/get.go +++ b/share/ipld/get.go @@ -7,7 +7,7 @@ import ( "sync/atomic" "github.com/gammazero/workerpool" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "go.opentelemetry.io/otel/attribute" diff --git a/share/ipld/get_shares.go b/share/ipld/get_shares.go index 0bed240fdc..3fda86941f 100644 --- a/share/ipld/get_shares.go +++ b/share/ipld/get_shares.go @@ -3,7 +3,7 @@ package ipld import ( "context" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" diff --git a/share/ipld/get_shares_test.go b/share/ipld/get_shares_test.go index dd6c3a6636..580efcb69b 100644 --- a/share/ipld/get_shares_test.go +++ b/share/ipld/get_shares_test.go @@ -11,13 +11,8 @@ import ( "testing" "time" - "github.com/ipfs/boxo/blockstore" - "github.com/ipfs/boxo/exchange/offline" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" - mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -35,7 +30,7 @@ func TestGetShare(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - bServ := mdutils.Bserv() + bServ := NewMemBlockservice() // generate random shares for the nmt shares := sharetest.RandShares(t, size*size) @@ -155,7 +150,7 @@ func removeRandShares(data [][]byte, d int) [][]byte { func TestGetSharesByNamespace(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) t.Cleanup(cancel) - bServ := mdutils.Bserv() + bServ := NewMemBlockservice() var tests = []struct { rawData []share.Share @@ -201,7 +196,7 @@ func TestGetSharesByNamespace(t *testing.T) { func TestCollectLeavesByNamespace_IncompleteData(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) t.Cleanup(cancel) - bServ := mdutils.Bserv() + bServ := NewMemBlockservice() shares := sharetest.RandShares(t, 16) @@ -245,7 +240,7 @@ func TestCollectLeavesByNamespace_IncompleteData(t *testing.T) { func TestCollectLeavesByNamespace_AbsentNamespaceId(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) t.Cleanup(cancel) - bServ := mdutils.Bserv() + bServ := NewMemBlockservice() shares := sharetest.RandShares(t, 1024) @@ -290,7 +285,7 @@ func TestCollectLeavesByNamespace_AbsentNamespaceId(t *testing.T) { func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) t.Cleanup(cancel) - bServ := mdutils.Bserv() + bServ := NewMemBlockservice() shares := sharetest.RandShares(t, 16) @@ -332,7 +327,7 @@ func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testi func TestGetSharesWithProofsByNamespace(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) t.Cleanup(cancel) - bServ := mdutils.Bserv() + bServ := NewMemBlockservice() var tests = []struct { rawData []share.Share @@ -431,13 +426,13 @@ func TestBatchSize(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(tt.origWidth)) defer cancel() - bs := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) + bs := NewMemBlockservice() randEds := edstest.RandEDS(t, tt.origWidth) - _, err := AddShares(ctx, randEds.FlattenedODS(), blockservice.New(bs, offline.Exchange(bs))) + _, err := AddShares(ctx, randEds.FlattenedODS(), bs) require.NoError(t, err) - out, err := bs.AllKeysChan(ctx) + out, err := bs.Blockstore().AllKeysChan(ctx) require.NoError(t, err) var count int diff --git a/share/ipld/namespace_data.go b/share/ipld/namespace_data.go index 38dfdb2169..a3f468ee47 100644 --- a/share/ipld/namespace_data.go +++ b/share/ipld/namespace_data.go @@ -7,7 +7,7 @@ import ( "sync" "sync/atomic" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "go.opentelemetry.io/otel/attribute" diff --git a/share/ipld/nmt.go b/share/ipld/nmt.go index e7d6b4b513..b43d786a5f 100644 --- a/share/ipld/nmt.go +++ b/share/ipld/nmt.go @@ -8,8 +8,8 @@ import ( "hash" "math/rand" + "github.com/ipfs/boxo/blockservice" blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" diff --git a/share/ipld/nmt_adder.go b/share/ipld/nmt_adder.go index d090c679d9..7ce52859b2 100644 --- a/share/ipld/nmt_adder.go +++ b/share/ipld/nmt_adder.go @@ -5,10 +5,10 @@ import ( "fmt" "sync" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" + "github.com/ipfs/boxo/ipld/merkledag" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" "github.com/celestiaorg/nmt" ) From da8b0e0744b67d65e59323c1072e2aa575dca0ee Mon Sep 17 00:00:00 2001 From: Ryan Date: Tue, 5 Sep 2023 12:02:23 +0200 Subject: [PATCH 2/2] chore: sort imports --- header/headertest/fraud/testing.go | 2 +- nodebuilder/tests/fraud_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/header/headertest/fraud/testing.go b/header/headertest/fraud/testing.go index 6e849b0ec6..e2ff13a4e0 100644 --- a/header/headertest/fraud/testing.go +++ b/header/headertest/fraud/testing.go @@ -2,10 +2,10 @@ package headerfraud import ( "context" - "github.com/ipfs/boxo/blockservice" "testing" "time" + "github.com/ipfs/boxo/blockservice" "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/libs/bytes" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" diff --git a/nodebuilder/tests/fraud_test.go b/nodebuilder/tests/fraud_test.go index a63d934588..95c702c0c0 100644 --- a/nodebuilder/tests/fraud_test.go +++ b/nodebuilder/tests/fraud_test.go @@ -2,10 +2,10 @@ package tests import ( "context" - "github.com/ipfs/go-datastore" "testing" "time" + "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer"