Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: update UnixFS/MFS with auto sharding #8114

Closed
wants to merge 18 commits into from
+111 −74
Closed

feat: update UnixFS/MFS with auto sharding #8114

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
c02f5f7
feat: update UnixFS/MFS with auto sharding
schomatis May 7, 2021
dac7742
WIP: ideas to fix sharness tests
schomatis May 7, 2021
a1c5b78
fix sharness
schomatis May 18, 2021
04dcb59
fix t0250 sharding and update unixfs with new-dir fix
schomatis May 19, 2021
c755b85
fix: update cids
schomatis May 19, 2021
3caa9da
test against modified interop repo for sharding changes
aschmahmann Oct 14, 2021
989467e
go mod tidy
aschmahmann Oct 14, 2021
5abe3dc
go fmt
aschmahmann Oct 14, 2021
7e395a5
update unixfs sharding path test
aschmahmann Oct 14, 2021
395652e
bump js-ipfs version used in interop tests
aschmahmann Oct 14, 2021
232e96e
update example deps
aschmahmann Oct 14, 2021
db96295
update sharness with un-sharding feature
schomatis Oct 20, 2021
168600f
[TEMP]: update to commit https://github.com/ipfs/go-unixfs/pull/99/co…
schomatis Oct 22, 2021
435fd73
remove just enough entries
schomatis Oct 22, 2021
374a2a0
fix unsharding sharness test
schomatis Oct 22, 2021
26e39aa
fix(UnixfsAPI): better handle link error
schomatis Oct 25, 2021
921abda
fix CIDv1, check against v0.10
schomatis Oct 25, 2021
56866cf
bump go-mfs and go-unixfs
aschmahmann Nov 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .circleci/main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ jobs:
command: |
npm init -y
npm install ipfs@^0.59.1
npm install ipfs-interop@^7.0.3
npm install ipfs/interop#fix/go-ipfs-automatic-sharding
npm install mocha-circleci-reporter@0.0.3
working_directory: ~/ipfs/go-ipfs/interop
- run:
Expand Down
12 changes: 6 additions & 6 deletions core/coreapi/test/path_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
"github.com/ipld/go-ipld-prime"
)


func TestPathUnixFSHAMTPartial(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
Expand All @@ -27,16 +26,17 @@ func TestPathUnixFSHAMTPartial(t *testing.T) {
a := apis[0]

// Setting this after instantiating the swarm so that it's not clobbered by loading the go-ipfs config
prevVal := uio.UseHAMTSharding
uio.UseHAMTSharding = true
// TODO: Either create a sharded directory manually, or make it such that it is automatically sized as sharded
prevVal := uio.HAMTShardingSize
uio.HAMTShardingSize = 1
defer func() {
uio.UseHAMTSharding = prevVal
uio.HAMTShardingSize = prevVal
}()

// Create and add a sharded directory
dir := make(map[string]files.Node)
// Make sure we have at least two levels of sharding
for i := 0; i < uio.DefaultShardWidth + 1; i++ {
for i := 0; i < uio.DefaultShardWidth+1; i++ {
dir[strconv.Itoa(i)] = files.NewBytesFile([]byte(strconv.Itoa(i)))
}

Expand Down Expand Up @@ -67,7 +67,7 @@ func TestPathUnixFSHAMTPartial(t *testing.T) {
for k := range dir {
// The node will go out to the (non-existent) network looking for the missing block. Make sure we're erroring
// because we exceeded the timeout on our query
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, time.Second * 1)
timeoutCtx, timeoutCancel := context.WithTimeout(ctx, time.Second*1)
_, err := a.ResolveNode(timeoutCtx, path.Join(r, k))
if err != nil {
if timeoutCtx.Err() == nil {
Expand Down
5 changes: 3 additions & 2 deletions core/coreapi/unixfs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,14 +218,15 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, opts ...options.Unixf

func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, settings *options.UnixfsLsSettings) coreiface.DirEntry {
if linkres.Err != nil {
return coreiface.DirEntry{Err: linkres.Err}
return coreiface.DirEntry{
Err: linkres.Err,
}
}

lnk := coreiface.DirEntry{
Name: linkres.Link.Name,
Cid: linkres.Link.Cid,
}

switch lnk.Cid.Type() {
case cid.Raw:
// No need to check with raw leaves
Expand Down
3 changes: 2 additions & 1 deletion core/node/groups.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/ipfs/go-ipfs/core/node/libp2p"
"github.com/ipfs/go-ipfs/p2p"

"github.com/alecthomas/units"
offline "github.com/ipfs/go-ipfs-exchange-offline"
offroute "github.com/ipfs/go-ipfs-routing/offline"
uio "github.com/ipfs/go-unixfs/io"
Expand Down Expand Up @@ -317,7 +318,7 @@ func IPFS(ctx context.Context, bcfg *BuildCfg) fx.Option {
}

// TEMP: setting global sharding switch here
uio.UseHAMTSharding = cfg.Experimental.ShardingEnabled
uio.HAMTShardingSize = int(256 * units.KiB)
Comment on lines 320 to +321
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems like this should live in go-unixfs since we'd like people to be using the automatic sharding unless they opt not to

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As it stands they will be using automatic sharding by default for go-ipfs, not for go-unixfs as a standalone library (independent of BitSwap, which is the one who introduce this restriction in the first place).

Do we also want this to be a default for go-unixfs as a library? (In that case we would set the HAMTShardingSize default value to 256KiB in unixfs and not here.)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we also want this to be a default for go-unixfs as a library? (In that case we would set the HAMTShardingSize default value to 256KiB in unixfs and not here.)

Yes, I think we should just put this in go-unixfs. If at some point that library starts taking sharding sizes in a non-global fashion we can deal with it there. The point of defaults is to give people something sane to start with if they're not deeply "in the know" so we might as well give it to go-unixfs consumers as well.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moving to UnixFS.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added in ipfs/go-unixfs@83ad983.

Will remove this after we update UnixFS/MFS dependencies with latest fixes.


return fx.Options(
bcfgOpts,
Expand Down
13 changes: 6 additions & 7 deletions docs/examples/go-ipfs-as-a-library/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM=
github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=
Expand Down Expand Up @@ -467,7 +468,6 @@ github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFq
github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C81I1NSHW1FxGew=
github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0=
github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
github.com/ipfs/go-ipfs-files v0.0.8 h1:8o0oFJkJ8UkO/ABl8T6ac6tKF3+NIpj67aAB6ZpusRg=
github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs=
github.com/ipfs/go-ipfs-keystore v0.0.2 h1:Fa9xg9IFD1VbiZtrNLzsD0GuELVHUFXCWF64kCPfEXU=
github.com/ipfs/go-ipfs-keystore v0.0.2/go.mod h1:H49tRmibOEs7gLMgbOsjC4dqh1u5e0R/SWuc2ScfgSo=
Expand Down Expand Up @@ -517,7 +517,6 @@ github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Ax
github.com/ipfs/go-log/v2 v2.3.0 h1:31Re/cPqFHpsRHgyVwjWADPoF0otB1WrjTy8ZFYwEZU=
github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g=
github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto=
github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
github.com/ipfs/go-merkledag v0.3.0/go.mod h1:4pymaZLhSLNVuiCITYrpViD6vmfZ/Ws4n/L9tfNv3S4=
github.com/ipfs/go-merkledag v0.3.1/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
Expand All @@ -527,8 +526,8 @@ github.com/ipfs/go-merkledag v0.4.0/go.mod h1:XshXBkhyeS63YNGisLL1uDSfuTyrQIxVUO
github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks=
github.com/ipfs/go-mfs v0.1.2 h1:DlelNSmH+yz/Riy0RjPKlooPg0KML4lXGdLw7uZkfAg=
github.com/ipfs/go-mfs v0.1.2/go.mod h1:T1QBiZPEpkPLzDqEJLNnbK55BVKVlNi2a+gVm4diFo0=
github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164 h1:0ATu9s5KktHhm8aYRSe1ysOJPik3dRwU/uag1Bcz+tg=
github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164/go.mod h1:A525zyeY2o078AoxhjJirOlDTXI1GnZxiYQnESGJ9WU=
github.com/ipfs/go-namesys v0.3.1 h1:DqmeXlVODejOyECAqoqhSB5JGRv8aRFhtG0oPDmxsMc=
github.com/ipfs/go-namesys v0.3.1/go.mod h1:/BL4xk8LP5Lq82AmaRKyxZv/eYRlumNiU9SZUe1Hlps=
github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno=
Expand All @@ -543,10 +542,10 @@ github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUn
github.com/ipfs/go-peertaskqueue v0.4.0 h1:x1hFgA4JOUJ3ntPfqLRu6v4k6kKL0p07r3RSg9JNyHI=
github.com/ipfs/go-peertaskqueue v0.4.0/go.mod h1:KL9F49hXJMoXCad8e5anivjN+kWdr+CyGcyh4K6doLc=
github.com/ipfs/go-pinning-service-http-client v0.1.0/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4=
github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s=
github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
github.com/ipfs/go-unixfs v0.2.5 h1:irj/WzIcgTBay48mSMUYDbKlIzIocXWcuUUsi5qOMOE=
github.com/ipfs/go-unixfs v0.2.5/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
github.com/ipfs/go-unixfs v0.2.6/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0=
github.com/ipfs/go-unixfs v0.2.7-0.20210519135750-3befc7e347fa h1:b2JM6ADtCk9+DgUmhwR15oxQSnyEucqNssy6GRfm+J4=
github.com/ipfs/go-unixfs v0.2.7-0.20210519135750-3befc7e347fa/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0=
github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s=
github.com/ipfs/go-unixfsnode v1.1.3 h1:IyqJBGIEvcHvll1wDDVIHOEVXnE+IH6tjzTWpZ6kGiI=
github.com/ipfs/go-unixfsnode v1.1.3/go.mod h1:ZZxUM5wXBC+G0Co9FjrYTOm+UlhZTjxLfRYdWY9veZ4=
Expand Down
5 changes: 3 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/ipfs/go-ipfs
require (
bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc
contrib.go.opencensus.io/exporter/prometheus v0.3.0
github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a
github.com/blang/semver/v4 v4.0.0
github.com/cheggaaa/pb v1.0.29
github.com/coreos/go-systemd/v22 v22.3.2
Expand Down Expand Up @@ -49,11 +50,11 @@ require (
github.com/ipfs/go-merkledag v0.4.0
github.com/ipfs/go-metrics-interface v0.0.1
github.com/ipfs/go-metrics-prometheus v0.0.2
github.com/ipfs/go-mfs v0.1.2
github.com/ipfs/go-mfs v0.1.3-0.20211112012225-23d6734eab23
github.com/ipfs/go-namesys v0.3.1
github.com/ipfs/go-path v0.1.2
github.com/ipfs/go-pinning-service-http-client v0.1.0
github.com/ipfs/go-unixfs v0.2.5
github.com/ipfs/go-unixfs v0.2.7-0.20211112011223-bd53b6a811b1
github.com/ipfs/go-unixfsnode v1.1.3
github.com/ipfs/go-verifcid v0.0.1
github.com/ipfs/interface-go-ipfs-core v0.5.1
Expand Down
14 changes: 8 additions & 6 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc=
github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM=
github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
Expand Down Expand Up @@ -385,6 +387,8 @@ github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod
github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/go-bitfield v1.0.0 h1:y/XHm2GEmD9wKngheWNNCNL0pzrWXZwCdQGv1ikXknQ=
github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus=
github.com/ipfs/go-bitswap v0.0.9/go.mod h1:kAPf5qgn2W2DrgAcscZ3HrM9qh4pH+X8Fkk3UPrwvis=
github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0=
github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs=
Expand Down Expand Up @@ -529,7 +533,6 @@ github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Ax
github.com/ipfs/go-log/v2 v2.3.0 h1:31Re/cPqFHpsRHgyVwjWADPoF0otB1WrjTy8ZFYwEZU=
github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g=
github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto=
github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
github.com/ipfs/go-merkledag v0.3.0/go.mod h1:4pymaZLhSLNVuiCITYrpViD6vmfZ/Ws4n/L9tfNv3S4=
github.com/ipfs/go-merkledag v0.3.1/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
Expand All @@ -540,8 +543,8 @@ github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fG
github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s=
github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks=
github.com/ipfs/go-mfs v0.1.2 h1:DlelNSmH+yz/Riy0RjPKlooPg0KML4lXGdLw7uZkfAg=
github.com/ipfs/go-mfs v0.1.2/go.mod h1:T1QBiZPEpkPLzDqEJLNnbK55BVKVlNi2a+gVm4diFo0=
github.com/ipfs/go-mfs v0.1.3-0.20211112012225-23d6734eab23 h1:RP+F0BONWIXRDb1x5QXk9PV7AnEosaKZCzU4RInsNvw=
github.com/ipfs/go-mfs v0.1.3-0.20211112012225-23d6734eab23/go.mod h1:+NT2mLpzr3LKkRZbjYSFjUTUzHAqJjPbBfnDsLJPJjw=
github.com/ipfs/go-namesys v0.3.1 h1:DqmeXlVODejOyECAqoqhSB5JGRv8aRFhtG0oPDmxsMc=
github.com/ipfs/go-namesys v0.3.1/go.mod h1:/BL4xk8LP5Lq82AmaRKyxZv/eYRlumNiU9SZUe1Hlps=
github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno=
Expand All @@ -557,10 +560,9 @@ github.com/ipfs/go-peertaskqueue v0.4.0 h1:x1hFgA4JOUJ3ntPfqLRu6v4k6kKL0p07r3RSg
github.com/ipfs/go-peertaskqueue v0.4.0/go.mod h1:KL9F49hXJMoXCad8e5anivjN+kWdr+CyGcyh4K6doLc=
github.com/ipfs/go-pinning-service-http-client v0.1.0 h1:Au0P4NglL5JfzhNSZHlZ1qra+IcJyO3RWMd9EYCwqSY=
github.com/ipfs/go-pinning-service-http-client v0.1.0/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4=
github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s=
github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
github.com/ipfs/go-unixfs v0.2.5 h1:irj/WzIcgTBay48mSMUYDbKlIzIocXWcuUUsi5qOMOE=
github.com/ipfs/go-unixfs v0.2.5/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
github.com/ipfs/go-unixfs v0.2.7-0.20211112011223-bd53b6a811b1 h1:FFSVXA9ns5IqwQRZUgq4GFk7qN1C9LbAwY6CJwk0I4Q=
github.com/ipfs/go-unixfs v0.2.7-0.20211112011223-bd53b6a811b1/go.mod h1:t8BWCW4OvTjcxQsX4e+GFroSZ5fCUXB5ywIMbw9eH/Y=
github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s=
github.com/ipfs/go-unixfsnode v1.1.3 h1:IyqJBGIEvcHvll1wDDVIHOEVXnE+IH6tjzTWpZ6kGiI=
github.com/ipfs/go-unixfsnode v1.1.3/go.mod h1:ZZxUM5wXBC+G0Co9FjrYTOm+UlhZTjxLfRYdWY9veZ4=
Expand Down
32 changes: 24 additions & 8 deletions test/sharness/t0250-files-api.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,21 +45,37 @@ verify_dir_contents() {
'
}

# Same as `t0260-sharding.sh`, we need a big directory now to trigger sharding.
test_expect_success "set up test data" '
mkdir big_dir
for i in `seq 6500` # just to be sure
do
echo $i > big_dir/`printf "file%06d" $i` # fixed length of 10 chars
done
'

test_sharding() {
local EXTRA ARGS
EXTRA=$1
ARGS=$2 # only applied to the initial directory

test_expect_success "make a directory $EXTRA" '
ipfs files mkdir $ARGS /foo
ipfs add $ARGS -r -q big_dir | tail -n1 > sharddir_cid &&
ipfs files cp /ipfs/`cat sharddir_cid` /foo
'

test_expect_success "can make 100 files in a directory $EXTRA" '
printf "" > list_exp_raw
for i in `seq 100 -1 1`
# Already present in the directory to trigger sharding
for i in `seq 6500` # just to be sure
do
echo `printf "file%06d" $i` >> list_exp_raw
done

for i in `seq 10100 -1 10000`
do
echo $i | ipfs files write --create /foo/file$i || return 1
echo file$i >> list_exp_raw
echo $i | ipfs files write --create /foo/`printf "file%06d" $i` || return 1
echo `printf "file%06d" $i` >> list_exp_raw
done
'
# Create the files in reverse (unsorted) order (`seq 100 -1 1`)
Expand All @@ -80,13 +96,13 @@ test_sharding() {
'

test_expect_success "can read a file from sharded directory $EXTRA" '
ipfs files read /foo/file65 > file_out &&
ipfs files read /foo/file000065 > file_out &&
echo "65" > file_exp &&
test_cmp file_out file_exp
'

test_expect_success "can pin a file from sharded directory $EXTRA" '
ipfs files stat --hash /foo/file42 > pin_file_hash &&
ipfs files stat --hash /foo/file000042 > pin_file_hash &&
ipfs pin add < pin_file_hash > pin_hash
'

Expand Down Expand Up @@ -817,10 +833,10 @@ test_expect_success "enable sharding in config" '

test_launch_ipfs_daemon_without_network

SHARD_HASH=QmPkwLJTYZRGPJ8Lazr9qPdrLmswPtUjaDbEpmR9jEh1se
SHARD_HASH=QmWSqxvnnHh6NL9ctzpwMiWg7iyCKTfcvLoQbCqC7qPQHw
test_sharding "(cidv0)"

SHARD_HASH=bafybeib46tpawg2d2hhlmmn2jvgio33wqkhlehxrem7wbfvqqikure37rm
SHARD_HASH=bafybeid7vaa7ywcd3mgrkdohpyqm5wxtmkwovyvxftzio3wuie3vlx4tcq
test_sharding "(cidv1 root)" "--cid-version=1"

test_kill_ipfs_daemon
Expand Down
Loading