Fix sharding memory growth, and fix resolver for unixfs paths

License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>

hamt/hamt.go

@@ -62,7 +62,7 @@ type HamtShard struct {
 
 // child can either be another shard, or a leaf node value
 type child interface {
-	Node() (node.Node, error)
+	Link() (*node.Link, error)
 	Label() string
 }
 
@@ -144,12 +144,12 @@ func (ds *HamtShard) Node() (node.Node, error) {
 		cindex := ds.indexForBitPos(i)
 		ch := ds.children[cindex]
 		if ch != nil {
-			cnd, err := ch.Node()
+			clnk, err := ch.Link()
 			if err != nil {
 				return nil, err
 			}
 
-			err = out.AddNodeLinkClean(ds.linkNamePrefix(i)+ch.Label(), cnd)
+			err = out.AddRawLink(ds.linkNamePrefix(i)+ch.Label(), clnk)
 			if err != nil {
 				return nil, err
 			}
@@ -188,10 +188,10 @@ func (ds *HamtShard) Node() (node.Node, error) {
 
 type shardValue struct {
 	key string
-	val node.Node
+	val *node.Link
 }
 
-func (sv *shardValue) Node() (node.Node, error) {
+func (sv *shardValue) Link() (*node.Link, error) {
 	return sv.val, nil
 }
 
@@ -214,7 +214,18 @@ func (ds *HamtShard) Label() string {
 // Set sets 'name' = nd in the HAMT
 func (ds *HamtShard) Set(ctx context.Context, name string, nd node.Node) error {
 	hv := &hashBits{b: hash([]byte(name))}
-	return ds.modifyValue(ctx, hv, name, nd)
+	_, err := ds.dserv.Add(nd)
+	if err != nil {
+		return err
+	}
+
+	lnk, err := node.MakeLink(nd)
+	if err != nil {
+		return err
+	}
+	lnk.Name = ds.linkNamePrefix(0) + name
+
+	return ds.modifyValue(ctx, hv, name, lnk)
 }
 
 // Remove deletes the named entry if it exists, this operation is idempotent.
@@ -226,13 +237,16 @@ func (ds *HamtShard) Remove(ctx context.Context, name string) error {
 func (ds *HamtShard) Find(ctx context.Context, name string) (node.Node, error) {
 	hv := &hashBits{b: hash([]byte(name))}
 
-	var out node.Node
+	var out *node.Link
 	err := ds.getValue(ctx, hv, name, func(sv *shardValue) error {
 		out = sv.val
 		return nil
 	})
+	if err != nil {
+		return nil, err
+	}
 
-	return out, err
+	return ds.dserv.Get(ctx, out.Cid)
 }
 
 // getChild returns the i'th child of this shard. If it is cached in the
@@ -291,9 +305,10 @@ func (ds *HamtShard) loadChild(ctx context.Context, i int) (child, error) {
 
 		c = cds
 	} else {
+		lnk2 := *lnk
 		c = &shardValue{
 			key: lnk.Name[ds.maxpadlen:],
-			val: nd,
+			val: &lnk2,
 		}
 	}
 
@@ -305,16 +320,32 @@ func (ds *HamtShard) setChild(i int, c child) {
 	ds.children[i] = c
 }
 
-func (ds *HamtShard) insertChild(idx int, key string, val node.Node) error {
-	if val == nil {
+func (ds *HamtShard) Link() (*node.Link, error) {
+	nd, err := ds.Node()
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = ds.dserv.Add(nd)
+	if err != nil {
+		return nil, err
+	}
+
+	return node.MakeLink(nd)
+}
+
+func (ds *HamtShard) insertChild(idx int, key string, lnk *node.Link) error {
+	if lnk == nil {
 		return os.ErrNotExist
 	}
 
 	i := ds.indexForBitPos(idx)
 	ds.bitfield.SetBit(ds.bitfield, idx, 1)
+
+	lnk.Name = ds.linkNamePrefix(idx) + key
 	sv := &shardValue{
 		key: key,
-		val: val,
+		val: lnk,
 	}
 
 	ds.children = append(ds.children[:i], append([]child{sv}, ds.children[i:]...)...)
@@ -370,11 +401,7 @@ func (ds *HamtShard) EnumLinks(ctx context.Context) ([]*node.Link, error) {
 
 func (ds *HamtShard) ForEachLink(ctx context.Context, f func(*node.Link) error) error {
 	return ds.walkTrie(ctx, func(sv *shardValue) error {
-		lnk, err := node.MakeLink(sv.val)
-		if err != nil {
-			return err
-		}
-
+		lnk := sv.val
 		lnk.Name = sv.key
 
 		return f(lnk)
@@ -414,7 +441,7 @@ func (ds *HamtShard) walkTrie(ctx context.Context, cb func(*shardValue) error) e
 	return nil
 }
 
-func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string, val node.Node) error {
+func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string, val *node.Link) error {
 	idx := hv.Next(ds.tableSizeLg2)
 
 	if ds.bitfield.Bit(idx) != 1 {

io/dirbuilder.go

@@ -48,10 +48,12 @@ func NewDirectory(dserv mdag.DAGService) *Directory {
 	return db
 }
 
+var ErrNotADir = fmt.Errorf("merkledag node was not a directory or shard")
+
 func NewDirectoryFromNode(dserv mdag.DAGService, nd node.Node) (*Directory, error) {
 	pbnd, ok := nd.(*mdag.ProtoNode)
 	if !ok {
-		return nil, mdag.ErrNotProtobuf
+		return nil, ErrNotADir
 	}
 
 	pbd, err := format.FromBytes(pbnd.Data())
@@ -76,7 +78,7 @@ func NewDirectoryFromNode(dserv mdag.DAGService, nd node.Node) (*Directory, erro
 			shard: shard,
 		}, nil
 	default:
-		return nil, fmt.Errorf("merkledag node was not a directory or shard")
+		return nil, ErrNotADir
 	}
 }