From c02f5f756fd2bf0132be5c9f74dd70d384baefc8 Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Fri, 7 May 2021 17:10:00 -0300
Subject: [PATCH 01/18] feat: update UnixFS/MFS with auto sharding

---
 core/node/groups.go | 3 ++-
 go.mod              | 5 +++--
 go.sum              | 4 ++++
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/core/node/groups.go b/core/node/groups.go
index 930a2151bef..f307e259780 100644
--- a/core/node/groups.go
+++ b/core/node/groups.go
@@ -16,6 +16,7 @@ import (
 	"github.com/ipfs/go-ipfs/core/node/libp2p"
 	"github.com/ipfs/go-ipfs/p2p"
 
+	"github.com/alecthomas/units"
 	offline "github.com/ipfs/go-ipfs-exchange-offline"
 	offroute "github.com/ipfs/go-ipfs-routing/offline"
 	uio "github.com/ipfs/go-unixfs/io"
@@ -317,7 +318,7 @@ func IPFS(ctx context.Context, bcfg *BuildCfg) fx.Option {
 	}
 
 	// TEMP: setting global sharding switch here
-	uio.UseHAMTSharding = cfg.Experimental.ShardingEnabled
+	uio.HAMTShardingSize = int(256 * units.KiB)
 
 	return fx.Options(
 		bcfgOpts,
diff --git a/go.mod b/go.mod
index 739d85be399..233ea388670 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,7 @@ module github.com/ipfs/go-ipfs
 require (
 	bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc
 	contrib.go.opencensus.io/exporter/prometheus v0.3.0
+	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d
 	github.com/blang/semver/v4 v4.0.0
 	github.com/cheggaaa/pb v1.0.29
 	github.com/coreos/go-systemd/v22 v22.3.2
@@ -49,11 +50,11 @@ require (
 	github.com/ipfs/go-merkledag v0.4.0
 	github.com/ipfs/go-metrics-interface v0.0.1
 	github.com/ipfs/go-metrics-prometheus v0.0.2
-	github.com/ipfs/go-mfs v0.1.2
+	github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164
 	github.com/ipfs/go-namesys v0.3.1
 	github.com/ipfs/go-path v0.1.2
 	github.com/ipfs/go-pinning-service-http-client v0.1.0
-	github.com/ipfs/go-unixfs v0.2.5
+	github.com/ipfs/go-unixfs v0.2.6
 	github.com/ipfs/go-unixfsnode v1.1.3
 	github.com/ipfs/go-verifcid v0.0.1
 	github.com/ipfs/interface-go-ipfs-core v0.5.1
diff --git a/go.sum b/go.sum
index 643d4206182..a2756f12dab 100644
--- a/go.sum
+++ b/go.sum
@@ -542,6 +542,8 @@ github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZa
 github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks=
 github.com/ipfs/go-mfs v0.1.2 h1:DlelNSmH+yz/Riy0RjPKlooPg0KML4lXGdLw7uZkfAg=
 github.com/ipfs/go-mfs v0.1.2/go.mod h1:T1QBiZPEpkPLzDqEJLNnbK55BVKVlNi2a+gVm4diFo0=
+github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164 h1:0ATu9s5KktHhm8aYRSe1ysOJPik3dRwU/uag1Bcz+tg=
+github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164/go.mod h1:A525zyeY2o078AoxhjJirOlDTXI1GnZxiYQnESGJ9WU=
 github.com/ipfs/go-namesys v0.3.1 h1:DqmeXlVODejOyECAqoqhSB5JGRv8aRFhtG0oPDmxsMc=
 github.com/ipfs/go-namesys v0.3.1/go.mod h1:/BL4xk8LP5Lq82AmaRKyxZv/eYRlumNiU9SZUe1Hlps=
 github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno=
@@ -561,6 +563,8 @@ github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NC
 github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
 github.com/ipfs/go-unixfs v0.2.5 h1:irj/WzIcgTBay48mSMUYDbKlIzIocXWcuUUsi5qOMOE=
 github.com/ipfs/go-unixfs v0.2.5/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
+github.com/ipfs/go-unixfs v0.2.6 h1:gq3U3T2vh8x6tXhfo3uSO3n+2z4yW0tYtNgVP/3sIyA=
+github.com/ipfs/go-unixfs v0.2.6/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0=
 github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s=
 github.com/ipfs/go-unixfsnode v1.1.3 h1:IyqJBGIEvcHvll1wDDVIHOEVXnE+IH6tjzTWpZ6kGiI=
 github.com/ipfs/go-unixfsnode v1.1.3/go.mod h1:ZZxUM5wXBC+G0Co9FjrYTOm+UlhZTjxLfRYdWY9veZ4=

From dac7742e85859df1c7af5e4db07223d1e5c47156 Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Fri, 7 May 2021 19:58:21 -0300
Subject: [PATCH 02/18] WIP: ideas to fix sharness tests

---
 test/sharness/t0260-sharding.sh | 69 +++++++++++++++++++++------------
 1 file changed, 45 insertions(+), 24 deletions(-)

diff --git a/test/sharness/t0260-sharding.sh b/test/sharness/t0260-sharding.sh
index 4270d688d06..dd3b6bee025 100755
--- a/test/sharness/t0260-sharding.sh
+++ b/test/sharness/t0260-sharding.sh
@@ -8,51 +8,71 @@ test_description="Test directory sharding"
 
 . lib/test-lib.sh
 
+# FIXME: We now have dropped the global binary option of either sharding nothing
+#  or sharding everything. We now have a threshold of 256 KiB (see `HAMTShardingSize`
+#  in core/node/groups.go) above which directories are sharded. The directory size
+#  is estimated as the size of each link (roughly entry name and CID byte length,
+#  normally 34 bytes). So we need 256 KiB / (34 + 10) ~ 6000 entries in the directory
+#  (estimating a fixed 10 char for each name) to trigger sharding.
+# We also need to update the SHARDED/UNSHARDED CIDs.
+
 test_expect_success "set up test data" '
-  mkdir testdata
-  for i in `seq 2000`
+  mkdir big_dir
+  for i in `seq 6500` # just to be sure
+  do
+    echo $i > big_dir/`printf "file%06d" $i`
+  done
+
+  mkdir small_dir
+  for i in `seq 100`
   do
-    echo $i > testdata/file$i
+    echo $i > small_dir/file$i
   done
 '
 
 test_add_large_dir() {
   exphash="$1"
+  input_dir="$2" # FIXME: added a new entry to switch between small and big_dir
+                 #  we need to update the calls.
   test_expect_success "ipfs add on very large directory succeeds" '
-    ipfs add -r -Q testdata > sharddir_out &&
+    ipfs add -r -Q $input_dir > sharddir_out &&
     echo "$exphash" > sharddir_exp &&
     test_cmp sharddir_exp sharddir_out
   '
   test_expect_success "ipfs get on very large directory succeeds" '
-    ipfs get -o testdata-out "$exphash" &&
-    test_cmp testdata testdata-out
+    ipfs get -o output_dir "$exphash" &&
+    test_cmp $input_dir output_dir
+    rm output_dir -r # FIXME: Cleaning the output directory because we now have
+                     #  two different directories and it seems `ipfs get` doesnt
+                     #  overwrite just adds files to the output dir.
   '
 }
 
 test_init_ipfs
 
-UNSHARDED="QmavrTrQG4VhoJmantURAYuw3bowq3E2WcvP36NRQDAC1N"
-test_add_large_dir "$UNSHARDED"
+# FIXME: Updating CID based just on the expected output of a failed test run. These
+#  need to be confirmed.
+UNSHARDED_SMALL="QmZedLGyvgWiyGQzAw2GpuLZeqzxmVcUQUbyRkVWx5DkxK"
+test_add_large_dir "$UNSHARDED_SMALL" small_dir
 
 test_launch_ipfs_daemon
 
-test_add_large_dir "$UNSHARDED"
+test_add_large_dir "$UNSHARDED_SMALL" small_dir
 
 test_kill_ipfs_daemon
 
-test_expect_success "enable sharding" '
-  ipfs config --json Experimental.ShardingEnabled true
-'
-
-SHARDED="QmSCJD1KYLhVVHqBK3YyXuoEqHt7vggyJhzoFYbT8v1XYL"
-test_add_large_dir "$SHARDED"
+SHARDED_BIG="Qmbgq1VC2KSWWhdFLvH6mapN1wuY67FZVPYbCGyR5TnYSv"
+test_add_large_dir "$SHARDED_BIG" big_dir
 
 test_launch_ipfs_daemon
 
-test_add_large_dir "$SHARDED"
+test_add_large_dir "$SHARDED_BIG" big_dir
 
 test_kill_ipfs_daemon
 
+# FIXME: This test no longer works, we have different directories for sharded
+#  and unsharded. We might have to duplicate the UNSHARDED/SHARDED for small
+#  and big dirs.
 test_expect_success "sharded and unsharded output look the same" '
   ipfs ls "$SHARDED" | sort > sharded_out &&
   ipfs ls "$UNSHARDED" | sort > unsharded_out &&
@@ -65,15 +85,16 @@ test_expect_success "ipfs cat error output the same" '
   test_cmp sharded_err unsharded_err
 '
 
+# FIXME: Should we replicate this test for the small (non sharded) directory as well?
 test_expect_success "'ipfs ls --resolve-type=false --size=false' admits missing block" '
-  ipfs ls "$SHARDED" | head -1 > first_file &&
-  ipfs ls --size=false "$SHARDED" | sort > sharded_out_nosize &&
+  ipfs ls "$SHARDED_BIG" | head -1 > first_file &&
+  ipfs ls --size=false "$SHARDED_BIG" | sort > sharded_out_nosize &&
   read -r HASH _ NAME <first_file &&
-  ipfs pin rm "$SHARDED" "$UNSHARDED" && # To allow us to remove the block
+  ipfs pin rm "$SHARDED_BIG" "$UNSHARDED" && # To allow us to remove the block
   ipfs block rm "$HASH" &&
-  test_expect_code 1 ipfs cat "$SHARDED/$NAME" &&
-  test_expect_code 1 ipfs ls "$SHARDED" &&
-  ipfs ls --resolve-type=false --size=false "$SHARDED" | sort > missing_out &&
+  test_expect_code 1 ipfs cat "$SHARDED_BIG/$NAME" &&
+  test_expect_code 1 ipfs ls "$SHARDED_BIG" &&
+  ipfs ls --resolve-type=false --size=false "$SHARDED_BIG" | sort > missing_out &&
   test_cmp sharded_out_nosize missing_out
 '
 
@@ -81,13 +102,13 @@ test_launch_ipfs_daemon
 
 test_expect_success "gateway can resolve sharded dirs" '
   echo 100 > expected &&
-  curl -sfo actual "http://127.0.0.1:$GWAY_PORT/ipfs/$SHARDED/file100" &&
+  curl -sfo actual "http://127.0.0.1:$GWAY_PORT/ipfs/$SHARDED_BIG/file100" &&
   test_cmp expected actual
 '
 
 test_expect_success "'ipfs resolve' can resolve sharded dirs" '
   echo /ipfs/QmZ3RfWk1u5LEGYLHA633B5TNJy3Du27K6Fny9wcxpowGS > expected &&
-  ipfs resolve "/ipfs/$SHARDED/file100" > actual &&
+  ipfs resolve "/ipfs/$SHARDED_BIG/file100" > actual &&
   test_cmp expected actual
 '
 

From a1c5b78ad85ade445bcacd5c8da1a6f10460c81d Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Tue, 18 May 2021 19:52:05 -0300
Subject: [PATCH 03/18] fix sharness

---
 test/sharness/t0260-sharding.sh | 88 +++++++++++++--------------------
 1 file changed, 35 insertions(+), 53 deletions(-)

diff --git a/test/sharness/t0260-sharding.sh b/test/sharness/t0260-sharding.sh
index dd3b6bee025..ff0df28cea4 100755
--- a/test/sharness/t0260-sharding.sh
+++ b/test/sharness/t0260-sharding.sh
@@ -8,32 +8,35 @@ test_description="Test directory sharding"
 
 . lib/test-lib.sh
 
-# FIXME: We now have dropped the global binary option of either sharding nothing
-#  or sharding everything. We now have a threshold of 256 KiB (see `HAMTShardingSize`
-#  in core/node/groups.go) above which directories are sharded. The directory size
-#  is estimated as the size of each link (roughly entry name and CID byte length,
-#  normally 34 bytes). So we need 256 KiB / (34 + 10) ~ 6000 entries in the directory
-#  (estimating a fixed 10 char for each name) to trigger sharding.
-# We also need to update the SHARDED/UNSHARDED CIDs.
-
+# We shard based on size with a threshold of 256 KiB (see `HAMTShardingSize`
+#  in core/node/groups.go) above which directories are sharded.
+# The directory size is estimated as the size of each link (roughly entry name,
+#  here of always 10 chars, and CID byte length, normally 34 bytes). So we need
+#  256 KiB / (34 + 10) ~ 6000 entries in the directory to trigger sharding.
+# We create then two directories: one above the threshold (big_dir) and one
+# below (small_dir), and hard-code the CIDs of their sharded and unsharded
+# codifications as IPFS directories.
 test_expect_success "set up test data" '
   mkdir big_dir
   for i in `seq 6500` # just to be sure
   do
-    echo $i > big_dir/`printf "file%06d" $i`
+    echo $i > big_dir/`printf "file%06d" $i` # fixed length of 10 chars
   done
 
   mkdir small_dir
   for i in `seq 100`
   do
-    echo $i > small_dir/file$i
+    echo $i > small_dir/`printf "file%06d" $i`
   done
 '
+# CID of big_dir/ which will be sharded.
+SHARDED="Qmbgq1VC2KSWWhdFLvH6mapN1wuY67FZVPYbCGyR5TnYSv"
+# CID of small_dir/ which will *not* be sharded.
+UNSHARDED="QmdBXmm4HRpUhyzzctbFvi2tLai3XFL1YjmE1qfpJe61NX"
 
 test_add_large_dir() {
   exphash="$1"
-  input_dir="$2" # FIXME: added a new entry to switch between small and big_dir
-                 #  we need to update the calls.
+  input_dir="$2"
   test_expect_success "ipfs add on very large directory succeeds" '
     ipfs add -r -Q $input_dir > sharddir_out &&
     echo "$exphash" > sharddir_exp &&
@@ -42,59 +45,37 @@ test_add_large_dir() {
   test_expect_success "ipfs get on very large directory succeeds" '
     ipfs get -o output_dir "$exphash" &&
     test_cmp $input_dir output_dir
-    rm output_dir -r # FIXME: Cleaning the output directory because we now have
-                     #  two different directories and it seems `ipfs get` doesnt
-                     #  overwrite just adds files to the output dir.
+    rm output_dir -r
   '
 }
 
 test_init_ipfs
 
-# FIXME: Updating CID based just on the expected output of a failed test run. These
-#  need to be confirmed.
-UNSHARDED_SMALL="QmZedLGyvgWiyGQzAw2GpuLZeqzxmVcUQUbyRkVWx5DkxK"
-test_add_large_dir "$UNSHARDED_SMALL" small_dir
+test_add_large_dir "$SHARDED" big_dir
+test_add_large_dir "$UNSHARDED" small_dir
 
 test_launch_ipfs_daemon
 
-test_add_large_dir "$UNSHARDED_SMALL" small_dir
+test_add_large_dir "$SHARDED" big_dir
+test_add_large_dir "$UNSHARDED" small_dir
 
 test_kill_ipfs_daemon
 
-SHARDED_BIG="Qmbgq1VC2KSWWhdFLvH6mapN1wuY67FZVPYbCGyR5TnYSv"
-test_add_large_dir "$SHARDED_BIG" big_dir
-
-test_launch_ipfs_daemon
-
-test_add_large_dir "$SHARDED_BIG" big_dir
-
-test_kill_ipfs_daemon
-
-# FIXME: This test no longer works, we have different directories for sharded
-#  and unsharded. We might have to duplicate the UNSHARDED/SHARDED for small
-#  and big dirs.
-test_expect_success "sharded and unsharded output look the same" '
-  ipfs ls "$SHARDED" | sort > sharded_out &&
-  ipfs ls "$UNSHARDED" | sort > unsharded_out &&
-  test_cmp sharded_out unsharded_out
-'
-
 test_expect_success "ipfs cat error output the same" '
   test_expect_code 1 ipfs cat "$SHARDED" 2> sharded_err &&
   test_expect_code 1 ipfs cat "$UNSHARDED" 2> unsharded_err &&
   test_cmp sharded_err unsharded_err
 '
 
-# FIXME: Should we replicate this test for the small (non sharded) directory as well?
 test_expect_success "'ipfs ls --resolve-type=false --size=false' admits missing block" '
-  ipfs ls "$SHARDED_BIG" | head -1 > first_file &&
-  ipfs ls --size=false "$SHARDED_BIG" | sort > sharded_out_nosize &&
+  ipfs ls "$SHARDED" | head -1 > first_file &&
+  ipfs ls --size=false "$SHARDED" | sort > sharded_out_nosize &&
   read -r HASH _ NAME <first_file &&
-  ipfs pin rm "$SHARDED_BIG" "$UNSHARDED" && # To allow us to remove the block
+  ipfs pin rm "$SHARDED" "$UNSHARDED" && # To allow us to remove the block
   ipfs block rm "$HASH" &&
-  test_expect_code 1 ipfs cat "$SHARDED_BIG/$NAME" &&
-  test_expect_code 1 ipfs ls "$SHARDED_BIG" &&
-  ipfs ls --resolve-type=false --size=false "$SHARDED_BIG" | sort > missing_out &&
+  test_expect_code 1 ipfs cat "$SHARDED/$NAME" &&
+  test_expect_code 1 ipfs ls "$SHARDED" &&
+  ipfs ls --resolve-type=false --size=false "$SHARDED" | sort > missing_out &&
   test_cmp sharded_out_nosize missing_out
 '
 
@@ -102,13 +83,13 @@ test_launch_ipfs_daemon
 
 test_expect_success "gateway can resolve sharded dirs" '
   echo 100 > expected &&
-  curl -sfo actual "http://127.0.0.1:$GWAY_PORT/ipfs/$SHARDED_BIG/file100" &&
+  curl -sfo actual "http://127.0.0.1:$GWAY_PORT/ipfs/$SHARDED/file000100" &&
   test_cmp expected actual
 '
 
 test_expect_success "'ipfs resolve' can resolve sharded dirs" '
   echo /ipfs/QmZ3RfWk1u5LEGYLHA633B5TNJy3Du27K6Fny9wcxpowGS > expected &&
-  ipfs resolve "/ipfs/$SHARDED_BIG/file100" > actual &&
+  ipfs resolve "/ipfs/$SHARDED/file000100" > actual &&
   test_cmp expected actual
 '
 
@@ -116,25 +97,26 @@ test_kill_ipfs_daemon
 
 test_add_large_dir_v1() {
   exphash="$1"
+  input_dir="$2"
   test_expect_success "ipfs add (CIDv1) on very large directory succeeds" '
-    ipfs add -r -Q --cid-version=1 testdata > sharddir_out &&
+    ipfs add -r -Q --cid-version=1 "$input_dir" > sharddir_out &&
     echo "$exphash" > sharddir_exp &&
     test_cmp sharddir_exp sharddir_out
   '
 
   test_expect_success "can access a path under the dir" '
-    ipfs cat "$exphash/file20" > file20_out &&
-    test_cmp testdata/file20 file20_out
+    ipfs cat "$exphash/file000020" > file20_out &&
+    test_cmp "$input_dir/file000020" file20_out
   '
 }
 
 # this hash implies the directory is CIDv1 and leaf entries are CIDv1 and raw
-SHARDEDV1="bafybeibiemewfzzdyhq2l74wrd6qj2oz42usjlktgnlqv4yfawgouaqn4u"
-test_add_large_dir_v1 "$SHARDEDV1"
+SHARDEDV1="bafybeif5mvotyb33vzncgkcorenaj6s7gwg7pgq3jmzuhosup3nrq2x7fi"
+test_add_large_dir_v1 "$SHARDEDV1" big_dir
 
 test_launch_ipfs_daemon
 
-test_add_large_dir_v1 "$SHARDEDV1"
+test_add_large_dir_v1 "$SHARDEDV1" big_dir
 
 test_kill_ipfs_daemon
 

From 04dcb592c43c9e86c1ea0753cb4b74dd1874145d Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Wed, 19 May 2021 09:41:54 -0300
Subject: [PATCH 04/18] fix t0250 sharding and update unixfs with new-dir fix

---
 go.mod                           |  2 +-
 go.sum                           |  2 ++
 test/sharness/t0250-files-api.sh | 32 ++++++++++++++++++++++++--------
 3 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/go.mod b/go.mod
index 233ea388670..dc8a32718c9 100644
--- a/go.mod
+++ b/go.mod
@@ -54,7 +54,7 @@ require (
 	github.com/ipfs/go-namesys v0.3.1
 	github.com/ipfs/go-path v0.1.2
 	github.com/ipfs/go-pinning-service-http-client v0.1.0
-	github.com/ipfs/go-unixfs v0.2.6
+	github.com/ipfs/go-unixfs v0.2.7-0.20210519135750-3befc7e347fa
 	github.com/ipfs/go-unixfsnode v1.1.3
 	github.com/ipfs/go-verifcid v0.0.1
 	github.com/ipfs/interface-go-ipfs-core v0.5.1
diff --git a/go.sum b/go.sum
index a2756f12dab..a0b6bcce5e9 100644
--- a/go.sum
+++ b/go.sum
@@ -565,6 +565,8 @@ github.com/ipfs/go-unixfs v0.2.5 h1:irj/WzIcgTBay48mSMUYDbKlIzIocXWcuUUsi5qOMOE=
 github.com/ipfs/go-unixfs v0.2.5/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
 github.com/ipfs/go-unixfs v0.2.6 h1:gq3U3T2vh8x6tXhfo3uSO3n+2z4yW0tYtNgVP/3sIyA=
 github.com/ipfs/go-unixfs v0.2.6/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0=
+github.com/ipfs/go-unixfs v0.2.7-0.20210519135750-3befc7e347fa h1:b2JM6ADtCk9+DgUmhwR15oxQSnyEucqNssy6GRfm+J4=
+github.com/ipfs/go-unixfs v0.2.7-0.20210519135750-3befc7e347fa/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0=
 github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s=
 github.com/ipfs/go-unixfsnode v1.1.3 h1:IyqJBGIEvcHvll1wDDVIHOEVXnE+IH6tjzTWpZ6kGiI=
 github.com/ipfs/go-unixfsnode v1.1.3/go.mod h1:ZZxUM5wXBC+G0Co9FjrYTOm+UlhZTjxLfRYdWY9veZ4=
diff --git a/test/sharness/t0250-files-api.sh b/test/sharness/t0250-files-api.sh
index 3276ff04d18..535630f97ea 100755
--- a/test/sharness/t0250-files-api.sh
+++ b/test/sharness/t0250-files-api.sh
@@ -45,21 +45,37 @@ verify_dir_contents() {
   '
 }
 
+# Same as `t0260-sharding.sh`, we need a big directory now to trigger sharding.
+test_expect_success "set up test data" '
+  mkdir big_dir
+  for i in `seq 6500` # just to be sure
+  do
+    echo $i > big_dir/`printf "file%06d" $i` # fixed length of 10 chars
+  done
+'
+
 test_sharding() {
   local EXTRA ARGS
   EXTRA=$1
   ARGS=$2 # only applied to the initial directory
 
   test_expect_success "make a directory $EXTRA" '
-    ipfs files mkdir $ARGS /foo
+    ipfs add $ARGS -r -q big_dir | tail -n1 > sharddir_cid &&
+    ipfs files cp /ipfs/`cat sharddir_cid` /foo
   '
 
   test_expect_success "can make 100 files in a directory $EXTRA" '
     printf "" > list_exp_raw
-    for i in `seq 100 -1 1`
+    # Already present in the directory to trigger sharding
+    for i in `seq 6500` # just to be sure
+    do
+      echo `printf "file%06d" $i` >> list_exp_raw
+    done
+
+    for i in `seq 10100 -1 10000`
     do
-      echo $i | ipfs files write --create /foo/file$i || return 1
-      echo file$i >> list_exp_raw
+      echo $i | ipfs files write --create /foo/`printf "file%06d" $i` || return 1
+      echo `printf "file%06d" $i` >> list_exp_raw
     done
   '
   # Create the files in reverse (unsorted) order (`seq 100 -1 1`)
@@ -80,13 +96,13 @@ test_sharding() {
   '
 
   test_expect_success "can read a file from sharded directory $EXTRA" '
-    ipfs files read /foo/file65 > file_out &&
+    ipfs files read /foo/file000065 > file_out &&
     echo "65" > file_exp &&
     test_cmp file_out file_exp
   '
 
   test_expect_success "can pin a file from sharded directory $EXTRA" '
-    ipfs files stat --hash /foo/file42 > pin_file_hash &&
+    ipfs files stat --hash /foo/file000042 > pin_file_hash &&
     ipfs pin add < pin_file_hash > pin_hash
   '
 
@@ -817,10 +833,10 @@ test_expect_success "enable sharding in config" '
 
 test_launch_ipfs_daemon_without_network
 
-SHARD_HASH=QmPkwLJTYZRGPJ8Lazr9qPdrLmswPtUjaDbEpmR9jEh1se
+SHARD_HASH=QmWSqxvnnHh6NL9ctzpwMiWg7iyCKTfcvLoQbCqC7qPQHw
 test_sharding "(cidv0)"
 
-SHARD_HASH=bafybeib46tpawg2d2hhlmmn2jvgio33wqkhlehxrem7wbfvqqikure37rm
+SHARD_HASH=bafybeid7vaa7ywcd3mgrkdohpyqm5wxtmkwovyvxftzio3wuie3vlx4tcq
 test_sharding "(cidv1 root)" "--cid-version=1"
 
 test_kill_ipfs_daemon

From c755b85bf18d8122a55535cb6666fa8c74e3071c Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Wed, 19 May 2021 12:42:12 -0300
Subject: [PATCH 05/18] fix: update cids

---
 test/sharness/t0260-sharding.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/sharness/t0260-sharding.sh b/test/sharness/t0260-sharding.sh
index ff0df28cea4..9b55a12e8c4 100755
--- a/test/sharness/t0260-sharding.sh
+++ b/test/sharness/t0260-sharding.sh
@@ -30,7 +30,7 @@ test_expect_success "set up test data" '
   done
 '
 # CID of big_dir/ which will be sharded.
-SHARDED="Qmbgq1VC2KSWWhdFLvH6mapN1wuY67FZVPYbCGyR5TnYSv"
+SHARDED="QmUj4SSHNz27z9t6DtZJiR56r17BWqwMrWCzBcR6hF2bq1"
 # CID of small_dir/ which will *not* be sharded.
 UNSHARDED="QmdBXmm4HRpUhyzzctbFvi2tLai3XFL1YjmE1qfpJe61NX"
 
@@ -111,7 +111,7 @@ test_add_large_dir_v1() {
 }
 
 # this hash implies the directory is CIDv1 and leaf entries are CIDv1 and raw
-SHARDEDV1="bafybeif5mvotyb33vzncgkcorenaj6s7gwg7pgq3jmzuhosup3nrq2x7fi"
+SHARDEDV1="bafybeie2tnyhaxbwkkzc44otilntecf55gvmnmnasjsppju7t6swhiw54e"
 test_add_large_dir_v1 "$SHARDEDV1" big_dir
 
 test_launch_ipfs_daemon

From 3caa9da94e2e940f30d48a97fac88e0bc6aa27dc Mon Sep 17 00:00:00 2001
From: Adin Schmahmann <adin.schmahmann@gmail.com>
Date: Wed, 13 Oct 2021 20:01:42 -0400
Subject: [PATCH 06/18] test against modified interop repo for sharding changes

---
 .circleci/main.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/main.yml b/.circleci/main.yml
index bdfe72f07e7..de12c0700cf 100644
--- a/.circleci/main.yml
+++ b/.circleci/main.yml
@@ -215,8 +215,8 @@ jobs:
           name: Installing dependencies
           command: |
             npm init -y
-            npm install ipfs@^0.59.1
-            npm install ipfs-interop@^7.0.3
+            npm install ipfs@^0.52.2
+            npm install ipfs/interop#fix/go-ipfs-automatic-sharding
             npm install mocha-circleci-reporter@0.0.3
           working_directory: ~/ipfs/go-ipfs/interop
       - run:

From 989467e761bbc397da3b9994efeddc4a10c9c23f Mon Sep 17 00:00:00 2001
From: Adin Schmahmann <adin.schmahmann@gmail.com>
Date: Wed, 13 Oct 2021 22:10:21 -0400
Subject: [PATCH 07/18] go mod tidy

---
 go.sum | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/go.sum b/go.sum
index a0b6bcce5e9..f04ecf1eeb6 100644
--- a/go.sum
+++ b/go.sum
@@ -64,6 +64,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM=
 github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=
@@ -529,7 +530,6 @@ github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Ax
 github.com/ipfs/go-log/v2 v2.3.0 h1:31Re/cPqFHpsRHgyVwjWADPoF0otB1WrjTy8ZFYwEZU=
 github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g=
 github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto=
-github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-merkledag v0.3.0/go.mod h1:4pymaZLhSLNVuiCITYrpViD6vmfZ/Ws4n/L9tfNv3S4=
 github.com/ipfs/go-merkledag v0.3.1/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
@@ -540,8 +540,6 @@ github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fG
 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
 github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s=
 github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks=
-github.com/ipfs/go-mfs v0.1.2 h1:DlelNSmH+yz/Riy0RjPKlooPg0KML4lXGdLw7uZkfAg=
-github.com/ipfs/go-mfs v0.1.2/go.mod h1:T1QBiZPEpkPLzDqEJLNnbK55BVKVlNi2a+gVm4diFo0=
 github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164 h1:0ATu9s5KktHhm8aYRSe1ysOJPik3dRwU/uag1Bcz+tg=
 github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164/go.mod h1:A525zyeY2o078AoxhjJirOlDTXI1GnZxiYQnESGJ9WU=
 github.com/ipfs/go-namesys v0.3.1 h1:DqmeXlVODejOyECAqoqhSB5JGRv8aRFhtG0oPDmxsMc=
@@ -559,11 +557,7 @@ github.com/ipfs/go-peertaskqueue v0.4.0 h1:x1hFgA4JOUJ3ntPfqLRu6v4k6kKL0p07r3RSg
 github.com/ipfs/go-peertaskqueue v0.4.0/go.mod h1:KL9F49hXJMoXCad8e5anivjN+kWdr+CyGcyh4K6doLc=
 github.com/ipfs/go-pinning-service-http-client v0.1.0 h1:Au0P4NglL5JfzhNSZHlZ1qra+IcJyO3RWMd9EYCwqSY=
 github.com/ipfs/go-pinning-service-http-client v0.1.0/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4=
-github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s=
 github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
-github.com/ipfs/go-unixfs v0.2.5 h1:irj/WzIcgTBay48mSMUYDbKlIzIocXWcuUUsi5qOMOE=
-github.com/ipfs/go-unixfs v0.2.5/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
-github.com/ipfs/go-unixfs v0.2.6 h1:gq3U3T2vh8x6tXhfo3uSO3n+2z4yW0tYtNgVP/3sIyA=
 github.com/ipfs/go-unixfs v0.2.6/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0=
 github.com/ipfs/go-unixfs v0.2.7-0.20210519135750-3befc7e347fa h1:b2JM6ADtCk9+DgUmhwR15oxQSnyEucqNssy6GRfm+J4=
 github.com/ipfs/go-unixfs v0.2.7-0.20210519135750-3befc7e347fa/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0=

From 5abe3dc824709be7850dc18378d8af0cb1cd58db Mon Sep 17 00:00:00 2001
From: Adin Schmahmann <adin.schmahmann@gmail.com>
Date: Wed, 13 Oct 2021 22:10:37 -0400
Subject: [PATCH 08/18] go fmt

---
 core/coreapi/test/path_test.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/core/coreapi/test/path_test.go b/core/coreapi/test/path_test.go
index cc7ba2efaef..fc0c5a21129 100644
--- a/core/coreapi/test/path_test.go
+++ b/core/coreapi/test/path_test.go
@@ -14,7 +14,6 @@ import (
 	"github.com/ipld/go-ipld-prime"
 )
 
-
 func TestPathUnixFSHAMTPartial(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -36,7 +35,7 @@ func TestPathUnixFSHAMTPartial(t *testing.T) {
 	// Create and add a sharded directory
 	dir := make(map[string]files.Node)
 	// Make sure we have at least two levels of sharding
-	for i := 0; i < uio.DefaultShardWidth + 1; i++ {
+	for i := 0; i < uio.DefaultShardWidth+1; i++ {
 		dir[strconv.Itoa(i)] = files.NewBytesFile([]byte(strconv.Itoa(i)))
 	}
 
@@ -67,7 +66,7 @@ func TestPathUnixFSHAMTPartial(t *testing.T) {
 	for k := range dir {
 		// The node will go out to the (non-existent) network looking for the missing block. Make sure we're erroring
 		// because we exceeded the timeout on our query
-		timeoutCtx, timeoutCancel := context.WithTimeout(ctx, time.Second * 1)
+		timeoutCtx, timeoutCancel := context.WithTimeout(ctx, time.Second*1)
 		_, err := a.ResolveNode(timeoutCtx, path.Join(r, k))
 		if err != nil {
 			if timeoutCtx.Err() == nil {

From 7e395a541d9c94df79a232b9e23083eeb765dfc8 Mon Sep 17 00:00:00 2001
From: Adin Schmahmann <adin.schmahmann@gmail.com>
Date: Wed, 13 Oct 2021 22:59:42 -0400
Subject: [PATCH 09/18] update unixfs sharding path test

---
 core/coreapi/test/path_test.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/core/coreapi/test/path_test.go b/core/coreapi/test/path_test.go
index fc0c5a21129..20ba2227d46 100644
--- a/core/coreapi/test/path_test.go
+++ b/core/coreapi/test/path_test.go
@@ -26,10 +26,11 @@ func TestPathUnixFSHAMTPartial(t *testing.T) {
 	a := apis[0]
 
 	// Setting this after instantiating the swarm so that it's not clobbered by loading the go-ipfs config
-	prevVal := uio.UseHAMTSharding
-	uio.UseHAMTSharding = true
+	// TODO: Either create a sharded directory manually, or make it such that it is automatically sized as sharded
+	prevVal := uio.HAMTShardingSize
+	uio.HAMTShardingSize = 1
 	defer func() {
-		uio.UseHAMTSharding = prevVal
+		uio.HAMTShardingSize = prevVal
 	}()
 
 	// Create and add a sharded directory

From 395652eb4bfe12b79368615d9723f39e5858a3df Mon Sep 17 00:00:00 2001
From: Adin Schmahmann <adin.schmahmann@gmail.com>
Date: Wed, 13 Oct 2021 23:00:27 -0400
Subject: [PATCH 10/18] bump js-ipfs version used in interop tests

---
 .circleci/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/main.yml b/.circleci/main.yml
index de12c0700cf..27bc3ccd38a 100644
--- a/.circleci/main.yml
+++ b/.circleci/main.yml
@@ -215,7 +215,7 @@ jobs:
           name: Installing dependencies
           command: |
             npm init -y
-            npm install ipfs@^0.52.2
+            npm install ipfs@^0.59.1
             npm install ipfs/interop#fix/go-ipfs-automatic-sharding
             npm install mocha-circleci-reporter@0.0.3
           working_directory: ~/ipfs/go-ipfs/interop

From 232e96ee603dcfc0b1ce936c107615557ea821c8 Mon Sep 17 00:00:00 2001
From: Adin Schmahmann <adin.schmahmann@gmail.com>
Date: Wed, 13 Oct 2021 23:57:57 -0400
Subject: [PATCH 11/18] update example deps

---
 docs/examples/go-ipfs-as-a-library/go.sum | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/docs/examples/go-ipfs-as-a-library/go.sum b/docs/examples/go-ipfs-as-a-library/go.sum
index b2cc10b62dc..3f9f801bbde 100644
--- a/docs/examples/go-ipfs-as-a-library/go.sum
+++ b/docs/examples/go-ipfs-as-a-library/go.sum
@@ -62,6 +62,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM=
 github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=
@@ -467,7 +468,6 @@ github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFq
 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C81I1NSHW1FxGew=
 github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0=
 github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
-github.com/ipfs/go-ipfs-files v0.0.8 h1:8o0oFJkJ8UkO/ABl8T6ac6tKF3+NIpj67aAB6ZpusRg=
 github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs=
 github.com/ipfs/go-ipfs-keystore v0.0.2 h1:Fa9xg9IFD1VbiZtrNLzsD0GuELVHUFXCWF64kCPfEXU=
 github.com/ipfs/go-ipfs-keystore v0.0.2/go.mod h1:H49tRmibOEs7gLMgbOsjC4dqh1u5e0R/SWuc2ScfgSo=
@@ -517,7 +517,6 @@ github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Ax
 github.com/ipfs/go-log/v2 v2.3.0 h1:31Re/cPqFHpsRHgyVwjWADPoF0otB1WrjTy8ZFYwEZU=
 github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g=
 github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto=
-github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-merkledag v0.3.0/go.mod h1:4pymaZLhSLNVuiCITYrpViD6vmfZ/Ws4n/L9tfNv3S4=
 github.com/ipfs/go-merkledag v0.3.1/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
@@ -527,8 +526,8 @@ github.com/ipfs/go-merkledag v0.4.0/go.mod h1:XshXBkhyeS63YNGisLL1uDSfuTyrQIxVUO
 github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
 github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks=
-github.com/ipfs/go-mfs v0.1.2 h1:DlelNSmH+yz/Riy0RjPKlooPg0KML4lXGdLw7uZkfAg=
-github.com/ipfs/go-mfs v0.1.2/go.mod h1:T1QBiZPEpkPLzDqEJLNnbK55BVKVlNi2a+gVm4diFo0=
+github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164 h1:0ATu9s5KktHhm8aYRSe1ysOJPik3dRwU/uag1Bcz+tg=
+github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164/go.mod h1:A525zyeY2o078AoxhjJirOlDTXI1GnZxiYQnESGJ9WU=
 github.com/ipfs/go-namesys v0.3.1 h1:DqmeXlVODejOyECAqoqhSB5JGRv8aRFhtG0oPDmxsMc=
 github.com/ipfs/go-namesys v0.3.1/go.mod h1:/BL4xk8LP5Lq82AmaRKyxZv/eYRlumNiU9SZUe1Hlps=
 github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno=
@@ -543,10 +542,10 @@ github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUn
 github.com/ipfs/go-peertaskqueue v0.4.0 h1:x1hFgA4JOUJ3ntPfqLRu6v4k6kKL0p07r3RSg9JNyHI=
 github.com/ipfs/go-peertaskqueue v0.4.0/go.mod h1:KL9F49hXJMoXCad8e5anivjN+kWdr+CyGcyh4K6doLc=
 github.com/ipfs/go-pinning-service-http-client v0.1.0/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4=
-github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s=
 github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
-github.com/ipfs/go-unixfs v0.2.5 h1:irj/WzIcgTBay48mSMUYDbKlIzIocXWcuUUsi5qOMOE=
-github.com/ipfs/go-unixfs v0.2.5/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
+github.com/ipfs/go-unixfs v0.2.6/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0=
+github.com/ipfs/go-unixfs v0.2.7-0.20210519135750-3befc7e347fa h1:b2JM6ADtCk9+DgUmhwR15oxQSnyEucqNssy6GRfm+J4=
+github.com/ipfs/go-unixfs v0.2.7-0.20210519135750-3befc7e347fa/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0=
 github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s=
 github.com/ipfs/go-unixfsnode v1.1.3 h1:IyqJBGIEvcHvll1wDDVIHOEVXnE+IH6tjzTWpZ6kGiI=
 github.com/ipfs/go-unixfsnode v1.1.3/go.mod h1:ZZxUM5wXBC+G0Co9FjrYTOm+UlhZTjxLfRYdWY9veZ4=

From db96295a70df37aca9599331fd1b696cb9464c8b Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Wed, 20 Oct 2021 12:39:09 -0300
Subject: [PATCH 12/18] update sharness with un-sharding feature

---
 test/sharness/t0260-sharding.sh | 42 +++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 15 deletions(-)

diff --git a/test/sharness/t0260-sharding.sh b/test/sharness/t0260-sharding.sh
index 9b55a12e8c4..004e81b0aeb 100755
--- a/test/sharness/t0260-sharding.sh
+++ b/test/sharness/t0260-sharding.sh
@@ -31,18 +31,21 @@ test_expect_success "set up test data" '
 '
 # CID of big_dir/ which will be sharded.
 SHARDED="QmUj4SSHNz27z9t6DtZJiR56r17BWqwMrWCzBcR6hF2bq1"
+# CID of big_dir/ once we remove half its entries and trigger a switch
+# back to a basic directory (un-sharding).
+UNSHAREDED="QmUj4SSHNz27z9t6DtZJiR56r17BWqwMrWCzBcR6hF2bq1"
 # CID of small_dir/ which will *not* be sharded.
-UNSHARDED="QmdBXmm4HRpUhyzzctbFvi2tLai3XFL1YjmE1qfpJe61NX"
+NOT_SHARDED="QmdBXmm4HRpUhyzzctbFvi2tLai3XFL1YjmE1qfpJe61NX"
 
-test_add_large_dir() {
+test_add_dir_with_sharding_enabled() {
   exphash="$1"
   input_dir="$2"
-  test_expect_success "ipfs add on very large directory succeeds" '
+  test_expect_success "ipfs add directory (with sharding enabled) succeeds" '
     ipfs add -r -Q $input_dir > sharddir_out &&
     echo "$exphash" > sharddir_exp &&
     test_cmp sharddir_exp sharddir_out
   '
-  test_expect_success "ipfs get on very large directory succeeds" '
+  test_expect_success "ipfs add directory (with sharding enabled) succeeds matches expected hash" '
     ipfs get -o output_dir "$exphash" &&
     test_cmp $input_dir output_dir
     rm output_dir -r
@@ -51,19 +54,28 @@ test_add_large_dir() {
 
 test_init_ipfs
 
-test_add_large_dir "$SHARDED" big_dir
-test_add_large_dir "$UNSHARDED" small_dir
+test_add_dir_with_sharding_enabled "$SHARDED" big_dir
+test_add_dir_with_sharding_enabled "$NOT_SHARDED" small_dir
 
 test_launch_ipfs_daemon
 
-test_add_large_dir "$SHARDED" big_dir
-test_add_large_dir "$UNSHARDED" small_dir
+test_add_dir_with_sharding_enabled "$SHARDED" big_dir
+test_add_dir_with_sharding_enabled "$NOT_SHARDED" small_dir
 
 test_kill_ipfs_daemon
 
+test_expect_success "remove entries from big_dir/" '
+  for i in `seq 3000` # just to be sure
+  do
+    rm big_dir/`printf "file%06d" $i`
+  done
+'
+
+test_add_dir_with_sharding_enabled "$UNSHAREDED" big_dir
+
 test_expect_success "ipfs cat error output the same" '
   test_expect_code 1 ipfs cat "$SHARDED" 2> sharded_err &&
-  test_expect_code 1 ipfs cat "$UNSHARDED" 2> unsharded_err &&
+  test_expect_code 1 ipfs cat "$NOT_SHARDED" 2> unsharded_err &&
   test_cmp sharded_err unsharded_err
 '
 
@@ -71,7 +83,7 @@ test_expect_success "'ipfs ls --resolve-type=false --size=false' admits missing
   ipfs ls "$SHARDED" | head -1 > first_file &&
   ipfs ls --size=false "$SHARDED" | sort > sharded_out_nosize &&
   read -r HASH _ NAME <first_file &&
-  ipfs pin rm "$SHARDED" "$UNSHARDED" && # To allow us to remove the block
+  ipfs pin rm "$SHARDED" "$NOT_SHARDED" && # To allow us to remove the block
   ipfs block rm "$HASH" &&
   test_expect_code 1 ipfs cat "$SHARDED/$NAME" &&
   test_expect_code 1 ipfs ls "$SHARDED" &&
@@ -95,7 +107,7 @@ test_expect_success "'ipfs resolve' can resolve sharded dirs" '
 
 test_kill_ipfs_daemon
 
-test_add_large_dir_v1() {
+test_add_dir_with_sharding_enabled_v1() {
   exphash="$1"
   input_dir="$2"
   test_expect_success "ipfs add (CIDv1) on very large directory succeeds" '
@@ -105,18 +117,18 @@ test_add_large_dir_v1() {
   '
 
   test_expect_success "can access a path under the dir" '
-    ipfs cat "$exphash/file000020" > file20_out &&
-    test_cmp "$input_dir/file000020" file20_out
+    ipfs cat "$exphash/file003020" > file3020_out &&
+    test_cmp "$input_dir/file003020" file3020_out
   '
 }
 
 # this hash implies the directory is CIDv1 and leaf entries are CIDv1 and raw
 SHARDEDV1="bafybeie2tnyhaxbwkkzc44otilntecf55gvmnmnasjsppju7t6swhiw54e"
-test_add_large_dir_v1 "$SHARDEDV1" big_dir
+test_add_dir_with_sharding_enabled_v1 "$SHARDEDV1" big_dir
 
 test_launch_ipfs_daemon
 
-test_add_large_dir_v1 "$SHARDEDV1" big_dir
+test_add_dir_with_sharding_enabled_v1 "$SHARDEDV1" big_dir
 
 test_kill_ipfs_daemon
 

From 168600f1aa28e3df60fcf878648ee06284f53329 Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Fri, 22 Oct 2021 14:22:18 -0300
Subject: [PATCH 13/18] [TEMP]: update to commit
 https://github.com/ipfs/go-unixfs/pull/99/commits/f7a8e99934ab225e1a7738d9773670f1036a1d05

---
 go.mod |  4 ++--
 go.sum | 11 ++++++-----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/go.mod b/go.mod
index dc8a32718c9..70f2bb5a7bd 100644
--- a/go.mod
+++ b/go.mod
@@ -50,11 +50,11 @@ require (
 	github.com/ipfs/go-merkledag v0.4.0
 	github.com/ipfs/go-metrics-interface v0.0.1
 	github.com/ipfs/go-metrics-prometheus v0.0.2
-	github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164
+	github.com/ipfs/go-mfs v0.1.3-0.20211022172037-3e82e1e60d6c
 	github.com/ipfs/go-namesys v0.3.1
 	github.com/ipfs/go-path v0.1.2
 	github.com/ipfs/go-pinning-service-http-client v0.1.0
-	github.com/ipfs/go-unixfs v0.2.7-0.20210519135750-3befc7e347fa
+	github.com/ipfs/go-unixfs v0.2.7-0.20211022171058-f7a8e99934ab
 	github.com/ipfs/go-unixfsnode v1.1.3
 	github.com/ipfs/go-verifcid v0.0.1
 	github.com/ipfs/interface-go-ipfs-core v0.5.1
diff --git a/go.sum b/go.sum
index f04ecf1eeb6..63078206fd3 100644
--- a/go.sum
+++ b/go.sum
@@ -386,6 +386,8 @@ github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod
 github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
+github.com/ipfs/go-bitfield v1.0.0 h1:y/XHm2GEmD9wKngheWNNCNL0pzrWXZwCdQGv1ikXknQ=
+github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus=
 github.com/ipfs/go-bitswap v0.0.9/go.mod h1:kAPf5qgn2W2DrgAcscZ3HrM9qh4pH+X8Fkk3UPrwvis=
 github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0=
 github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs=
@@ -540,8 +542,8 @@ github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fG
 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
 github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s=
 github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks=
-github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164 h1:0ATu9s5KktHhm8aYRSe1ysOJPik3dRwU/uag1Bcz+tg=
-github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164/go.mod h1:A525zyeY2o078AoxhjJirOlDTXI1GnZxiYQnESGJ9WU=
+github.com/ipfs/go-mfs v0.1.3-0.20211022172037-3e82e1e60d6c h1:abIhXTK5Nj2E+bUSTOXH4HykCU+U6Vi4k9Uhnzc5HSc=
+github.com/ipfs/go-mfs v0.1.3-0.20211022172037-3e82e1e60d6c/go.mod h1:xDArQRVSrt4Zy9puzsSJAfWoEzDVCpXV0G1ah0kJcxQ=
 github.com/ipfs/go-namesys v0.3.1 h1:DqmeXlVODejOyECAqoqhSB5JGRv8aRFhtG0oPDmxsMc=
 github.com/ipfs/go-namesys v0.3.1/go.mod h1:/BL4xk8LP5Lq82AmaRKyxZv/eYRlumNiU9SZUe1Hlps=
 github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno=
@@ -558,9 +560,8 @@ github.com/ipfs/go-peertaskqueue v0.4.0/go.mod h1:KL9F49hXJMoXCad8e5anivjN+kWdr+
 github.com/ipfs/go-pinning-service-http-client v0.1.0 h1:Au0P4NglL5JfzhNSZHlZ1qra+IcJyO3RWMd9EYCwqSY=
 github.com/ipfs/go-pinning-service-http-client v0.1.0/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4=
 github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
-github.com/ipfs/go-unixfs v0.2.6/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0=
-github.com/ipfs/go-unixfs v0.2.7-0.20210519135750-3befc7e347fa h1:b2JM6ADtCk9+DgUmhwR15oxQSnyEucqNssy6GRfm+J4=
-github.com/ipfs/go-unixfs v0.2.7-0.20210519135750-3befc7e347fa/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0=
+github.com/ipfs/go-unixfs v0.2.7-0.20211022171058-f7a8e99934ab h1:KvRvkoYP9PZRrw3v09PGwibhb0m353la1HvGJaRDK38=
+github.com/ipfs/go-unixfs v0.2.7-0.20211022171058-f7a8e99934ab/go.mod h1:4P7unwkBeL3rhfluLhnVKc+OXmFXlcJRfdgv7U1G/qA=
 github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s=
 github.com/ipfs/go-unixfsnode v1.1.3 h1:IyqJBGIEvcHvll1wDDVIHOEVXnE+IH6tjzTWpZ6kGiI=
 github.com/ipfs/go-unixfsnode v1.1.3/go.mod h1:ZZxUM5wXBC+G0Co9FjrYTOm+UlhZTjxLfRYdWY9veZ4=

From 435fd739a9b9d5373775152b102a9babe1ac6154 Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Fri, 22 Oct 2021 15:26:39 -0300
Subject: [PATCH 14/18] remove just enough entries

---
 test/sharness/t0260-sharding.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/sharness/t0260-sharding.sh b/test/sharness/t0260-sharding.sh
index 004e81b0aeb..31f3b922dc9 100755
--- a/test/sharness/t0260-sharding.sh
+++ b/test/sharness/t0260-sharding.sh
@@ -18,7 +18,7 @@ test_description="Test directory sharding"
 # codifications as IPFS directories.
 test_expect_success "set up test data" '
   mkdir big_dir
-  for i in `seq 6500` # just to be sure
+  for i in `seq 5960` # Just above the number of entries that trigger sharding for 256KiB
   do
     echo $i > big_dir/`printf "file%06d" $i` # fixed length of 10 chars
   done
@@ -33,7 +33,7 @@ test_expect_success "set up test data" '
 SHARDED="QmUj4SSHNz27z9t6DtZJiR56r17BWqwMrWCzBcR6hF2bq1"
 # CID of big_dir/ once we remove half its entries and trigger a switch
 # back to a basic directory (un-sharding).
-UNSHAREDED="QmUj4SSHNz27z9t6DtZJiR56r17BWqwMrWCzBcR6hF2bq1"
+UNSHAREDED="QmbvMyr8D2GDwuBkyJhbVWkrrVaJfFQhwUKFYqWnhkEiwg"
 # CID of small_dir/ which will *not* be sharded.
 NOT_SHARDED="QmdBXmm4HRpUhyzzctbFvi2tLai3XFL1YjmE1qfpJe61NX"
 
@@ -64,8 +64,8 @@ test_add_dir_with_sharding_enabled "$NOT_SHARDED" small_dir
 
 test_kill_ipfs_daemon
 
-test_expect_success "remove entries from big_dir/" '
-  for i in `seq 3000` # just to be sure
+test_expect_success "remove a few entries from big_dir/" '
+  for i in `seq 5` # just to be sure
   do
     rm big_dir/`printf "file%06d" $i`
   done

From 374a2a04b9274061f617bfde38671204ef37716a Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Fri, 22 Oct 2021 15:55:35 -0300
Subject: [PATCH 15/18] fix unsharding sharness test

---
 test/sharness/t0260-sharding.sh | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/test/sharness/t0260-sharding.sh b/test/sharness/t0260-sharding.sh
index 31f3b922dc9..75f7a615d4c 100755
--- a/test/sharness/t0260-sharding.sh
+++ b/test/sharness/t0260-sharding.sh
@@ -30,10 +30,10 @@ test_expect_success "set up test data" '
   done
 '
 # CID of big_dir/ which will be sharded.
-SHARDED="QmUj4SSHNz27z9t6DtZJiR56r17BWqwMrWCzBcR6hF2bq1"
+SHARDED="QmWfjnRWRvdvYezQWnfbvrvY7JjrpevsE9cato1x76UqGr"
 # CID of big_dir/ once we remove half its entries and trigger a switch
 # back to a basic directory (un-sharding).
-UNSHAREDED="QmbvMyr8D2GDwuBkyJhbVWkrrVaJfFQhwUKFYqWnhkEiwg"
+UNSHAREDED="QmbVxi5zDdzytrjdufUejM92JsWj8wGVmukk6tiPce3p1m"
 # CID of small_dir/ which will *not* be sharded.
 NOT_SHARDED="QmdBXmm4HRpUhyzzctbFvi2tLai3XFL1YjmE1qfpJe61NX"
 
@@ -64,15 +64,17 @@ test_add_dir_with_sharding_enabled "$NOT_SHARDED" small_dir
 
 test_kill_ipfs_daemon
 
-test_expect_success "remove a few entries from big_dir/" '
-  for i in `seq 5` # just to be sure
+test_expect_success "remove a few entries from big_dir/ to trigger unsharding" '
+  ipfs files cp /ipfs/$SHARDED /big_dir &&
+  for i in `seq 5`
   do
-    rm big_dir/`printf "file%06d" $i`
-  done
+    ipfs files rm /big_dir/`printf "file%06d" $i`
+  done &&
+  ipfs files stat --hash /big_dir > unshard_dir_hash &&
+  echo "$UNSHAREDED" > unshard_exp &&
+  test_cmp unshard_exp unshard_dir_hash
 '
 
-test_add_dir_with_sharding_enabled "$UNSHAREDED" big_dir
-
 test_expect_success "ipfs cat error output the same" '
   test_expect_code 1 ipfs cat "$SHARDED" 2> sharded_err &&
   test_expect_code 1 ipfs cat "$NOT_SHARDED" 2> unsharded_err &&

From 26e39aad9f528f32b0434261ac3567c02121b877 Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Mon, 25 Oct 2021 14:19:56 -0300
Subject: [PATCH 16/18] fix(UnixfsAPI): better handle link error

---
 core/coreapi/unixfs.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go
index 6146f807a71..50ced95fd15 100644
--- a/core/coreapi/unixfs.go
+++ b/core/coreapi/unixfs.go
@@ -218,14 +218,15 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, opts ...options.Unixf
 
 func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, settings *options.UnixfsLsSettings) coreiface.DirEntry {
 	if linkres.Err != nil {
-		return coreiface.DirEntry{Err: linkres.Err}
+		return coreiface.DirEntry{
+			Err:  linkres.Err,
+		}
 	}
 
 	lnk := coreiface.DirEntry{
 		Name: linkres.Link.Name,
 		Cid:  linkres.Link.Cid,
 	}
-
 	switch lnk.Cid.Type() {
 	case cid.Raw:
 		// No need to check with raw leaves

From 921abda47c19b0c4955ffbccd12b1257f21d3bd2 Mon Sep 17 00:00:00 2001
From: Lucas Molas <schomatis@gmail.com>
Date: Mon, 25 Oct 2021 15:11:12 -0300
Subject: [PATCH 17/18] fix CIDv1, check against v0.10

---
 test/sharness/t0260-sharding.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/sharness/t0260-sharding.sh b/test/sharness/t0260-sharding.sh
index 75f7a615d4c..1a1c046661f 100755
--- a/test/sharness/t0260-sharding.sh
+++ b/test/sharness/t0260-sharding.sh
@@ -125,7 +125,7 @@ test_add_dir_with_sharding_enabled_v1() {
 }
 
 # this hash implies the directory is CIDv1 and leaf entries are CIDv1 and raw
-SHARDEDV1="bafybeie2tnyhaxbwkkzc44otilntecf55gvmnmnasjsppju7t6swhiw54e"
+SHARDEDV1="bafybeiglv7wdwjqaqxjs4nk3uo4vvldhmmod5vkinvgwihyrv6bdzuzbbm"
 test_add_dir_with_sharding_enabled_v1 "$SHARDEDV1" big_dir
 
 test_launch_ipfs_daemon

From 56866cf1d2c96889c2a5522e954020448831c302 Mon Sep 17 00:00:00 2001
From: Adin Schmahmann <adin.schmahmann@gmail.com>
Date: Thu, 11 Nov 2021 20:32:14 -0500
Subject: [PATCH 18/18] bump go-mfs and go-unixfs

---
 go.mod |  6 +++---
 go.sum | 11 ++++++-----
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/go.mod b/go.mod
index 70f2bb5a7bd..3cbfa9fee03 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,7 @@ module github.com/ipfs/go-ipfs
 require (
 	bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc
 	contrib.go.opencensus.io/exporter/prometheus v0.3.0
-	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d
+	github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a
 	github.com/blang/semver/v4 v4.0.0
 	github.com/cheggaaa/pb v1.0.29
 	github.com/coreos/go-systemd/v22 v22.3.2
@@ -50,11 +50,11 @@ require (
 	github.com/ipfs/go-merkledag v0.4.0
 	github.com/ipfs/go-metrics-interface v0.0.1
 	github.com/ipfs/go-metrics-prometheus v0.0.2
-	github.com/ipfs/go-mfs v0.1.3-0.20211022172037-3e82e1e60d6c
+	github.com/ipfs/go-mfs v0.1.3-0.20211112012225-23d6734eab23
 	github.com/ipfs/go-namesys v0.3.1
 	github.com/ipfs/go-path v0.1.2
 	github.com/ipfs/go-pinning-service-http-client v0.1.0
-	github.com/ipfs/go-unixfs v0.2.7-0.20211022171058-f7a8e99934ab
+	github.com/ipfs/go-unixfs v0.2.7-0.20211112011223-bd53b6a811b1
 	github.com/ipfs/go-unixfsnode v1.1.3
 	github.com/ipfs/go-verifcid v0.0.1
 	github.com/ipfs/interface-go-ipfs-core v0.5.1
diff --git a/go.sum b/go.sum
index 63078206fd3..145cd25e17e 100644
--- a/go.sum
+++ b/go.sum
@@ -64,8 +64,9 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc=
+github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
 github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM=
 github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
@@ -542,8 +543,8 @@ github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fG
 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
 github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s=
 github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks=
-github.com/ipfs/go-mfs v0.1.3-0.20211022172037-3e82e1e60d6c h1:abIhXTK5Nj2E+bUSTOXH4HykCU+U6Vi4k9Uhnzc5HSc=
-github.com/ipfs/go-mfs v0.1.3-0.20211022172037-3e82e1e60d6c/go.mod h1:xDArQRVSrt4Zy9puzsSJAfWoEzDVCpXV0G1ah0kJcxQ=
+github.com/ipfs/go-mfs v0.1.3-0.20211112012225-23d6734eab23 h1:RP+F0BONWIXRDb1x5QXk9PV7AnEosaKZCzU4RInsNvw=
+github.com/ipfs/go-mfs v0.1.3-0.20211112012225-23d6734eab23/go.mod h1:+NT2mLpzr3LKkRZbjYSFjUTUzHAqJjPbBfnDsLJPJjw=
 github.com/ipfs/go-namesys v0.3.1 h1:DqmeXlVODejOyECAqoqhSB5JGRv8aRFhtG0oPDmxsMc=
 github.com/ipfs/go-namesys v0.3.1/go.mod h1:/BL4xk8LP5Lq82AmaRKyxZv/eYRlumNiU9SZUe1Hlps=
 github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno=
@@ -560,8 +561,8 @@ github.com/ipfs/go-peertaskqueue v0.4.0/go.mod h1:KL9F49hXJMoXCad8e5anivjN+kWdr+
 github.com/ipfs/go-pinning-service-http-client v0.1.0 h1:Au0P4NglL5JfzhNSZHlZ1qra+IcJyO3RWMd9EYCwqSY=
 github.com/ipfs/go-pinning-service-http-client v0.1.0/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4=
 github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
-github.com/ipfs/go-unixfs v0.2.7-0.20211022171058-f7a8e99934ab h1:KvRvkoYP9PZRrw3v09PGwibhb0m353la1HvGJaRDK38=
-github.com/ipfs/go-unixfs v0.2.7-0.20211022171058-f7a8e99934ab/go.mod h1:4P7unwkBeL3rhfluLhnVKc+OXmFXlcJRfdgv7U1G/qA=
+github.com/ipfs/go-unixfs v0.2.7-0.20211112011223-bd53b6a811b1 h1:FFSVXA9ns5IqwQRZUgq4GFk7qN1C9LbAwY6CJwk0I4Q=
+github.com/ipfs/go-unixfs v0.2.7-0.20211112011223-bd53b6a811b1/go.mod h1:t8BWCW4OvTjcxQsX4e+GFroSZ5fCUXB5ywIMbw9eH/Y=
 github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s=
 github.com/ipfs/go-unixfsnode v1.1.3 h1:IyqJBGIEvcHvll1wDDVIHOEVXnE+IH6tjzTWpZ6kGiI=
 github.com/ipfs/go-unixfsnode v1.1.3/go.mod h1:ZZxUM5wXBC+G0Co9FjrYTOm+UlhZTjxLfRYdWY9veZ4=