Add automatic sharding/unsharding tests (#8547)

* test: add tests for automatic sharding and unsharding
* refactor: changed some naming in the sharding sharness tests to make more sense

Co-authored-by: Adin Schmahmann <adin.schmahmann@gmail.com>

test/sharness/t0250-files-api.sh

@@ -858,4 +858,70 @@ test_sharding "(cidv1 root)" "--cid-version=1"
 
 test_kill_ipfs_daemon
 
+# Test automatic sharding and unsharding
+
+# We shard based on size with a threshold of 256 KiB (see config file docs)
+# above which directories are sharded.
+#
+# The directory size is estimated as the size of each link. Links are roughly
+# the entry name + the CID byte length (e.g. 34 bytes for a CIDv0). So for
+# entries of length 10 we need 256 KiB / (34 + 10) ~ 6000 entries in the
+# directory to trigger sharding.
+test_expect_success "set up automatic sharding/unsharding data" '
+  mkdir big_dir
+  for i in `seq 5960` # Just above the number of entries that trigger sharding for 256KiB
+  do
+    echo $i > big_dir/`printf "file%06d" $i` # fixed length of 10 chars
+  done
+'
+
+# TODO: This does not need to report an error https://github.com/ipfs/go-ipfs/issues/8088
+test_expect_failure "reset automatic sharding" '
+  ipfs config --json Internal.UnixFSShardingSizeThreshold null
+'
+
+test_launch_ipfs_daemon_without_network
+
+LARGE_SHARDED="QmWfjnRWRvdvYezQWnfbvrvY7JjrpevsE9cato1x76UqGr"
+LARGE_MINUS_5_UNSHARDED="QmbVxi5zDdzytrjdufUejM92JsWj8wGVmukk6tiPce3p1m"
+
+test_add_large_sharded_dir() {
+  exphash="$1"
+  test_expect_success "ipfs add on directory succeeds" '
+    ipfs add -r -Q big_dir > shardbigdir_out &&
+    echo "$exphash" > shardbigdir_exp &&
+    test_cmp shardbigdir_exp shardbigdir_out
+  '
+
+  test_expect_success "can access a path under the dir" '
+    ipfs cat "$exphash/file000030" > file30_out &&
+    test_cmp big_dir/file000030 file30_out
+  '
+}
+
+test_add_large_sharded_dir "$LARGE_SHARDED"
+
+test_expect_success "remove a few entries from big_dir/ to trigger unsharding" '
+  ipfs files cp /ipfs/"$LARGE_SHARDED" /big_dir &&
+  for i in `seq 5`
+  do
+    ipfs files rm /big_dir/`printf "file%06d" $i`
+  done &&
+  ipfs files stat --hash /big_dir > unshard_dir_hash &&
+  echo "$LARGE_MINUS_5_UNSHARDED" > unshard_exp &&
+  test_cmp unshard_exp unshard_dir_hash
+'
+
+test_expect_success "add a few entries to big_dir/ to retrigger sharding" '
+  for i in `seq 5`
+  do
+    ipfs files cp /ipfs/"$LARGE_SHARDED"/`printf "file%06d" $i` /big_dir/`printf "file%06d" $i`
+  done &&
+  ipfs files stat --hash /big_dir > shard_dir_hash &&
+  echo "$LARGE_SHARDED" > shard_exp &&
+  test_cmp shard_exp shard_dir_hash
+'
+
+test_kill_ipfs_daemon
+
 test_done

test/sharness/t0260-sharding.sh

@@ -16,14 +16,14 @@ test_expect_success "set up test data" '
   done
 '
 
-test_add_large_dir() {
+test_add_dir() {
   exphash="$1"
-  test_expect_success "ipfs add on very large directory succeeds" '
+  test_expect_success "ipfs add on directory succeeds" '
     ipfs add -r -Q testdata > sharddir_out &&
     echo "$exphash" > sharddir_exp &&
     test_cmp sharddir_exp sharddir_out
   '
-  test_expect_success "ipfs get on very large directory succeeds" '
+  test_expect_success "ipfs get on directory succeeds" '
     ipfs get -o testdata-out "$exphash" &&
     test_cmp testdata testdata-out
   '
@@ -32,24 +32,29 @@ test_add_large_dir() {
 test_init_ipfs
 
 UNSHARDED="QmavrTrQG4VhoJmantURAYuw3bowq3E2WcvP36NRQDAC1N"
-test_add_large_dir "$UNSHARDED"
+
+test_expect_success "force sharding off" '
+ipfs config --json Internal.UnixFSShardingSizeThreshold "\"1G\""
+'
+
+test_add_dir "$UNSHARDED"
 
 test_launch_ipfs_daemon
 
-test_add_large_dir "$UNSHARDED"
+test_add_dir "$UNSHARDED"
 
 test_kill_ipfs_daemon
 
-test_expect_success "enable sharding" '
+test_expect_success "force sharding on" '
   ipfs config --json Internal.UnixFSShardingSizeThreshold "\"1B\""
 '
 
 SHARDED="QmSCJD1KYLhVVHqBK3YyXuoEqHt7vggyJhzoFYbT8v1XYL"
-test_add_large_dir "$SHARDED"
+test_add_dir "$SHARDED"
 
 test_launch_ipfs_daemon
 
-test_add_large_dir "$SHARDED"
+test_add_dir "$SHARDED"
 
 test_kill_ipfs_daemon
 
@@ -93,9 +98,9 @@ test_expect_success "'ipfs resolve' can resolve sharded dirs" '
 
 test_kill_ipfs_daemon
 
-test_add_large_dir_v1() {
+test_add_dir_v1() {
   exphash="$1"
-  test_expect_success "ipfs add (CIDv1) on very large directory succeeds" '
+  test_expect_success "ipfs add (CIDv1) on directory succeeds" '
     ipfs add -r -Q --cid-version=1 testdata > sharddir_out &&
     echo "$exphash" > sharddir_exp &&
     test_cmp sharddir_exp sharddir_out
@@ -109,11 +114,11 @@ test_add_large_dir_v1() {
 
 # this hash implies the directory is CIDv1 and leaf entries are CIDv1 and raw
 SHARDEDV1="bafybeibiemewfzzdyhq2l74wrd6qj2oz42usjlktgnlqv4yfawgouaqn4u"
-test_add_large_dir_v1 "$SHARDEDV1"
+test_add_dir_v1 "$SHARDEDV1"
 
 test_launch_ipfs_daemon
 
-test_add_large_dir_v1 "$SHARDEDV1"
+test_add_dir_v1 "$SHARDEDV1"
 
 test_kill_ipfs_daemon