diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3c34288 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +*~ +*.log + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool +*.out diff --git a/dsindex/error.go b/dsindex/error.go new file mode 100644 index 0000000..f3b685b --- /dev/null +++ b/dsindex/error.go @@ -0,0 +1,8 @@ +package dsindex + +import "errors" + +var ( + ErrEmptyKey = errors.New("key is empty") + ErrEmptyValue = errors.New("value is empty") +) diff --git a/dsindex/indexer.go b/dsindex/indexer.go new file mode 100644 index 0000000..e48af2e --- /dev/null +++ b/dsindex/indexer.go @@ -0,0 +1,285 @@ +// Package dsindex provides secondary indexing functionality for a datastore. +package dsindex + +import ( + "context" + "fmt" + "path" + + ds "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + "github.com/ipfs/go-datastore/query" + "github.com/multiformats/go-multibase" +) + +// Indexer maintains a secondary index. An index is a collection of key-value +// mappings where the key is the secondary index that maps to one or more +// values, where each value is a unique key being indexed. +type Indexer interface { + // Add adds the specified value to the key + Add(ctx context.Context, key, value string) error + + // Delete deletes the specified value from the key. If the value is not in + // the datastore, this method returns no error. + Delete(ctx context.Context, key, value string) error + + // DeleteKey deletes all values in the given key. If a key is not in the + // datastore, this method returns no error. Returns a count of values that + // were deleted. + DeleteKey(ctx context.Context, key string) (count int, err error) + + // DeleteAll deletes all keys managed by this Indexer. Returns a count of + // the values that were deleted. + DeleteAll(ctx context.Context) (count int, err error) + + // ForEach calls the function for each value in the specified key, until + // there are no more values, or until the function returns false. If key + // is empty string, then all keys are iterated. + ForEach(ctx context.Context, key string, fn func(key, value string) bool) error + + // HasValue determines if the key contains the specified value + HasValue(ctx context.Context, key, value string) (bool, error) + + // HasAny determines if any value is in the specified key. If key is + // empty string, then all values are searched. + HasAny(ctx context.Context, key string) (bool, error) + + // Search returns all values for the given key + Search(ctx context.Context, key string) (values []string, err error) +} + +// indexer is a simple implementation of Indexer. This implementation relies +// on the underlying data store to support efficient querying by prefix. +// +// TODO: Consider adding caching +type indexer struct { + dstore ds.Datastore +} + +// New creates a new datastore index. All indexes are stored under the +// specified index name. +// +// To persist the actions of calling Indexer functions, it is necessary to call +// dstore.Sync. +func New(dstore ds.Datastore, name ds.Key) Indexer { + return &indexer{ + dstore: namespace.Wrap(dstore, name), + } +} + +func (x *indexer) Add(ctx context.Context, key, value string) error { + if key == "" { + return ErrEmptyKey + } + if value == "" { + return ErrEmptyValue + } + dsKey := ds.NewKey(encode(key)).ChildString(encode(value)) + return x.dstore.Put(dsKey, []byte{}) +} + +func (x *indexer) Delete(ctx context.Context, key, value string) error { + if key == "" { + return ErrEmptyKey + } + if value == "" { + return ErrEmptyValue + } + return x.dstore.Delete(ds.NewKey(encode(key)).ChildString(encode(value))) +} + +func (x *indexer) DeleteKey(ctx context.Context, key string) (int, error) { + if key == "" { + return 0, ErrEmptyKey + } + return x.deletePrefix(ctx, encode(key)) +} + +func (x *indexer) DeleteAll(ctx context.Context) (int, error) { + return x.deletePrefix(ctx, "") +} + +func (x *indexer) ForEach(ctx context.Context, key string, fn func(key, value string) bool) error { + if key != "" { + key = encode(key) + } + + q := query.Query{ + Prefix: key, + KeysOnly: true, + } + results, err := x.dstore.Query(q) + if err != nil { + return err + } + + for { + r, ok := results.NextSync() + if !ok { + break + } + if r.Error != nil { + err = r.Error + break + } + if ctx.Err() != nil { + err = ctx.Err() + break + } + ent := r.Entry + decIdx, err := decode(path.Base(path.Dir(ent.Key))) + if err != nil { + err = fmt.Errorf("cannot decode index: %v", err) + break + } + decKey, err := decode(path.Base(ent.Key)) + if err != nil { + err = fmt.Errorf("cannot decode key: %v", err) + break + } + if !fn(decIdx, decKey) { + break + } + } + results.Close() + + return err +} + +func (x *indexer) HasValue(ctx context.Context, key, value string) (bool, error) { + if key == "" { + return false, ErrEmptyKey + } + if value == "" { + return false, ErrEmptyValue + } + return x.dstore.Has(ds.NewKey(encode(key)).ChildString(encode(value))) +} + +func (x *indexer) HasAny(ctx context.Context, key string) (bool, error) { + var any bool + err := x.ForEach(ctx, key, func(key, value string) bool { + any = true + return false + }) + return any, err +} + +func (x *indexer) Search(ctx context.Context, key string) ([]string, error) { + if key == "" { + return nil, ErrEmptyKey + } + ents, err := x.queryPrefix(ctx, encode(key)) + if err != nil { + return nil, err + } + if len(ents) == 0 { + return nil, nil + } + + values := make([]string, len(ents)) + for i := range ents { + values[i], err = decode(path.Base(ents[i].Key)) + if err != nil { + return nil, fmt.Errorf("cannot decode value: %v", err) + } + } + return values, nil +} + +// SyncIndex synchronizes the keys in the target Indexer to match those of the +// ref Indexer. This function does not change this indexer's key root (name +// passed into New). +func SyncIndex(ctx context.Context, ref, target Indexer) (bool, error) { + // Build reference index map + refs := map[string]string{} + err := ref.ForEach(ctx, "", func(key, value string) bool { + refs[value] = key + return true + }) + if err != nil { + return false, err + } + if len(refs) == 0 { + return false, nil + } + + // Compare current indexes + dels := map[string]string{} + err = target.ForEach(ctx, "", func(key, value string) bool { + refKey, ok := refs[value] + if ok && refKey == key { + // same in both; delete from refs, do not add to dels + delete(refs, value) + } else { + dels[value] = key + } + return true + }) + if err != nil { + return false, err + } + + // Items in dels are keys that no longer exist + for value, key := range dels { + err = target.Delete(ctx, key, value) + if err != nil { + return false, err + } + } + + // What remains in refs are keys that need to be added + for value, key := range refs { + err = target.Add(ctx, key, value) + if err != nil { + return false, err + } + } + + return len(refs) != 0 || len(dels) != 0, nil +} + +func (x *indexer) deletePrefix(ctx context.Context, prefix string) (int, error) { + ents, err := x.queryPrefix(ctx, prefix) + if err != nil { + return 0, err + } + + for i := range ents { + err = x.dstore.Delete(ds.NewKey(ents[i].Key)) + if err != nil { + return 0, err + } + } + + return len(ents), nil +} + +func (x *indexer) queryPrefix(ctx context.Context, prefix string) ([]query.Entry, error) { + q := query.Query{ + Prefix: prefix, + KeysOnly: true, + } + results, err := x.dstore.Query(q) + if err != nil { + return nil, err + } + return results.Rest() +} + +func encode(data string) string { + encData, err := multibase.Encode(multibase.Base64url, []byte(data)) + if err != nil { + // programming error; using unsupported encoding + panic(err.Error()) + } + return encData +} + +func decode(data string) (string, error) { + _, b, err := multibase.Decode(data) + if err != nil { + return "", err + } + return string(b), nil +} diff --git a/dsindex/indexer_test.go b/dsindex/indexer_test.go new file mode 100644 index 0000000..45372c6 --- /dev/null +++ b/dsindex/indexer_test.go @@ -0,0 +1,286 @@ +package dsindex + +import ( + "context" + "testing" + + ds "github.com/ipfs/go-datastore" +) + +func createIndexer() Indexer { + dstore := ds.NewMapDatastore() + nameIndex := New(dstore, ds.NewKey("/data/nameindex")) + + ctx := context.Background() + nameIndex.Add(ctx, "alice", "a1") + nameIndex.Add(ctx, "bob", "b1") + nameIndex.Add(ctx, "bob", "b2") + nameIndex.Add(ctx, "cathy", "c1") + + return nameIndex +} + +func TestAdd(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nameIndex := createIndexer() + err := nameIndex.Add(ctx, "someone", "s1") + if err != nil { + t.Fatal(err) + } + err = nameIndex.Add(ctx, "someone", "s1") + if err != nil { + t.Fatal(err) + } + + err = nameIndex.Add(ctx, "", "noindex") + if err != ErrEmptyKey { + t.Fatal("unexpected error:", err) + } + + err = nameIndex.Add(ctx, "nokey", "") + if err != ErrEmptyValue { + t.Fatal("unexpected error:", err) + } +} + +func TestHasValue(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nameIndex := createIndexer() + + ok, err := nameIndex.HasValue(ctx, "bob", "b1") + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("missing index") + } + + ok, err = nameIndex.HasValue(ctx, "bob", "b3") + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatal("should not have index") + } + + _, err = nameIndex.HasValue(ctx, "", "b1") + if err != ErrEmptyKey { + t.Fatal("unexpected error:", err) + } + + _, err = nameIndex.HasValue(ctx, "bob", "") + if err != ErrEmptyValue { + t.Fatal("unexpected error:", err) + } +} + +func TestHasAny(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nameIndex := createIndexer() + + ok, err := nameIndex.HasAny(ctx, "nothere") + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatal("should return false") + } + + for _, idx := range []string{"alice", "bob", ""} { + ok, err = nameIndex.HasAny(ctx, idx) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("missing index", idx) + } + } + + count, err := nameIndex.DeleteAll(ctx) + if err != nil { + t.Fatal(err) + } + if count != 4 { + t.Fatal("expected 4 deletions") + } + + ok, err = nameIndex.HasAny(ctx, "") + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatal("should return false") + } +} + +func TestForEach(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nameIndex := createIndexer() + + found := make(map[string]struct{}) + err := nameIndex.ForEach(ctx, "bob", func(key, value string) bool { + found[value] = struct{}{} + return true + }) + if err != nil { + t.Fatal(err) + } + + for _, value := range []string{"b1", "b2"} { + _, ok := found[value] + if !ok { + t.Fatal("missing key for value", value) + } + } + + values := map[string]string{} + err = nameIndex.ForEach(ctx, "", func(key, value string) bool { + values[value] = key + return true + }) + if err != nil { + t.Fatal(err) + } + if len(values) != 4 { + t.Fatal("expected 4 keys") + } + + if values["a1"] != "alice" { + t.Error("expected a1: alice") + } + if values["b1"] != "bob" { + t.Error("expected b1: bob") + } + if values["b2"] != "bob" { + t.Error("expected b2: bob") + } + if values["c1"] != "cathy" { + t.Error("expected c1: cathy") + } +} + +func TestSearch(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nameIndex := createIndexer() + + ids, err := nameIndex.Search(ctx, "bob") + if err != nil { + t.Fatal(err) + } + if len(ids) != 2 { + t.Fatal("wrong number of ids - expected 2 got", ids) + } + for _, id := range ids { + if id != "b1" && id != "b2" { + t.Fatal("wrong value in id set") + } + } + if ids[0] == ids[1] { + t.Fatal("duplicate id") + } + + ids, err = nameIndex.Search(ctx, "cathy") + if err != nil { + t.Fatal(err) + } + if len(ids) != 1 || ids[0] != "c1" { + t.Fatal("wrong ids") + } + + ids, err = nameIndex.Search(ctx, "amit") + if err != nil { + t.Fatal(err) + } + if len(ids) != 0 { + t.Fatal("unexpected ids returned") + } +} + +func TestDelete(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nameIndex := createIndexer() + + err := nameIndex.Delete(ctx, "bob", "b3") + if err != nil { + t.Fatal(err) + } + + err = nameIndex.Delete(ctx, "alice", "a1") + if err != nil { + t.Fatal(err) + } + + ok, err := nameIndex.HasValue(ctx, "alice", "a1") + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatal("index key should have been deleted") + } + + count, err := nameIndex.DeleteKey(ctx, "bob") + if err != nil { + t.Fatal(err) + } + if count != 2 { + t.Fatal("wrong deleted count") + } + ok, _ = nameIndex.HasValue(ctx, "bob", "b1") + if ok { + t.Fatal("index not deleted") + } +} + +func TestSyncIndex(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nameIndex := createIndexer() + + dstore := ds.NewMapDatastore() + refIndex := New(dstore, ds.NewKey("/ref")) + refIndex.Add(ctx, "alice", "a1") + refIndex.Add(ctx, "cathy", "zz") + refIndex.Add(ctx, "dennis", "d1") + + changed, err := SyncIndex(ctx, refIndex, nameIndex) + if err != nil { + t.Fatal(err) + } + if !changed { + t.Error("change was not indicated") + } + + // Create map of id->index in sync target + syncs := map[string]string{} + err = nameIndex.ForEach(ctx, "", func(key, value string) bool { + syncs[value] = key + return true + }) + if err != nil { + t.Fatal(err) + } + + // Iterate items in sync source and make sure they appear in target + var itemCount int + err = refIndex.ForEach(ctx, "", func(key, value string) bool { + itemCount++ + syncKey, ok := syncs[value] + if !ok || key != syncKey { + t.Fatal("key", key, "-->", value, "was not synced") + } + return true + }) + if err != nil { + t.Fatal(err) + } + + if itemCount != len(syncs) { + t.Fatal("different number of items in sync source and target") + } +} diff --git a/dspinner/pin.go b/dspinner/pin.go new file mode 100644 index 0000000..5fd65e7 --- /dev/null +++ b/dspinner/pin.go @@ -0,0 +1,961 @@ +// Package dspinner implements structures and methods to keep track of +// which objects a user wants to keep stored locally. This implementation +// stores pin data in a datastore. +package dspinner + +import ( + "context" + "errors" + "fmt" + "path" + "sync" + + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" + ipfspinner "github.com/ipfs/go-ipfs-pinner" + "github.com/ipfs/go-ipfs-pinner/dsindex" + ipld "github.com/ipfs/go-ipld-format" + logging "github.com/ipfs/go-log" + mdag "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-merkledag/dagutils" + "github.com/polydawn/refmt/cbor" + "github.com/polydawn/refmt/obj/atlas" +) + +const ( + basePath = "/pins" + pinKeyPath = "/pins/pin" + indexKeyPath = "/pins/index" + dirtyKeyPath = "/pins/state/dirty" +) + +var ( + // ErrNotPinned is returned when trying to unpin items that are not pinned. + ErrNotPinned = errors.New("not pinned or pinned indirectly") + + log logging.StandardLogger = logging.Logger("pin") + + linkDirect, linkRecursive string + + pinCidDIndexPath string + pinCidRIndexPath string + pinNameIndexPath string + + dirtyKey = ds.NewKey(dirtyKeyPath) + + pinAtl atlas.Atlas +) + +func init() { + directStr, ok := ipfspinner.ModeToString(ipfspinner.Direct) + if !ok { + panic("could not find Direct pin enum") + } + linkDirect = directStr + + recursiveStr, ok := ipfspinner.ModeToString(ipfspinner.Recursive) + if !ok { + panic("could not find Recursive pin enum") + } + linkRecursive = recursiveStr + + pinCidRIndexPath = path.Join(indexKeyPath, "cidRindex") + pinCidDIndexPath = path.Join(indexKeyPath, "cidDindex") + pinNameIndexPath = path.Join(indexKeyPath, "nameIndex") + + pinAtl = atlas.MustBuild( + atlas.BuildEntry(pin{}).StructMap(). + AddField("Cid", atlas.StructMapEntry{SerialName: "cid"}). + AddField("Metadata", atlas.StructMapEntry{SerialName: "metadata", OmitEmpty: true}). + AddField("Mode", atlas.StructMapEntry{SerialName: "mode"}). + AddField("Name", atlas.StructMapEntry{SerialName: "name", OmitEmpty: true}). + Complete(), + atlas.BuildEntry(cid.Cid{}).Transform(). + TransformMarshal(atlas.MakeMarshalTransformFunc(func(live cid.Cid) ([]byte, error) { return live.MarshalBinary() })). + TransformUnmarshal(atlas.MakeUnmarshalTransformFunc(func(serializable []byte) (cid.Cid, error) { + c := cid.Cid{} + err := c.UnmarshalBinary(serializable) + if err != nil { + return cid.Cid{}, err + } + return c, nil + })).Complete(), + ) + pinAtl = pinAtl.WithMapMorphism(atlas.MapMorphism{KeySortMode: atlas.KeySortMode_Strings}) +} + +// pinner implements the Pinner interface +type pinner struct { + lock sync.RWMutex + + dserv ipld.DAGService + dstore ds.Datastore + + cidDIndex dsindex.Indexer + cidRIndex dsindex.Indexer + nameIndex dsindex.Indexer + + clean int64 + dirty int64 +} + +var _ ipfspinner.Pinner = (*pinner)(nil) + +type pin struct { + Id string + Cid cid.Cid + Metadata map[string]interface{} + Mode ipfspinner.Mode + Name string +} + +func (p *pin) dsKey() ds.Key { + return ds.NewKey(path.Join(pinKeyPath, p.Id)) +} + +func newPin(c cid.Cid, mode ipfspinner.Mode, name string) *pin { + return &pin{ + Id: ds.RandomKey().String(), + Cid: c, + Name: name, + Mode: mode, + } +} + +type syncDAGService interface { + ipld.DAGService + Sync() error +} + +// New creates a new pinner and loads its keysets from the given datastore. If +// there is no data present in the datastore, then an empty pinner is returned. +func New(ctx context.Context, dstore ds.Datastore, dserv ipld.DAGService) (ipfspinner.Pinner, error) { + p := &pinner{ + cidDIndex: dsindex.New(dstore, ds.NewKey(pinCidDIndexPath)), + cidRIndex: dsindex.New(dstore, ds.NewKey(pinCidRIndexPath)), + nameIndex: dsindex.New(dstore, ds.NewKey(pinNameIndexPath)), + dserv: dserv, + dstore: dstore, + } + + data, err := dstore.Get(dirtyKey) + if err != nil { + if err == ds.ErrNotFound { + return p, nil + } + return nil, fmt.Errorf("cannot load dirty flag: %v", err) + } + if data[0] == 1 { + p.dirty = 1 + + pins, err := p.loadAllPins(ctx) + if err != nil { + return nil, fmt.Errorf("cannot load pins: %v", err) + } + + err = p.rebuildIndexes(ctx, pins) + if err != nil { + return nil, fmt.Errorf("cannot rebuild indexes: %v", err) + } + } + + return p, nil +} + +// Pin the given node, optionally recursive +func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { + err := p.dserv.Add(ctx, node) + if err != nil { + return err + } + + c := node.Cid() + cidKey := c.KeyString() + + p.lock.Lock() + defer p.lock.Unlock() + + if recurse { + found, err := p.cidRIndex.HasAny(ctx, cidKey) + if err != nil { + return err + } + if found { + return nil + } + + dirtyBefore := p.dirty + + // temporary unlock to fetch the entire graph + p.lock.Unlock() + // Fetch graph starting at node identified by cid + err = mdag.FetchGraph(ctx, c, p.dserv) + p.lock.Lock() + if err != nil { + return err + } + + // Only look again if something has changed. + if p.dirty != dirtyBefore { + found, err = p.cidRIndex.HasAny(ctx, cidKey) + if err != nil { + return err + } + if found { + return nil + } + } + + // TODO: remove this to support multiple pins per CID + found, err = p.cidDIndex.HasAny(ctx, cidKey) + if err != nil { + return err + } + if found { + p.removePinsForCid(ctx, c, ipfspinner.Direct) + } + + _, err = p.addPin(ctx, c, ipfspinner.Recursive, "") + if err != nil { + return err + } + } else { + found, err := p.cidRIndex.HasAny(ctx, cidKey) + if err != nil { + return err + } + if found { + return fmt.Errorf("%s already pinned recursively", c.String()) + } + + _, err = p.addPin(ctx, c, ipfspinner.Direct, "") + if err != nil { + return err + } + } + return nil +} + +func (p *pinner) addPin(ctx context.Context, c cid.Cid, mode ipfspinner.Mode, name string) (string, error) { + // Create new pin and store in datastore + pp := newPin(c, mode, name) + + // Serialize pin + pinData, err := encodePin(pp) + if err != nil { + return "", fmt.Errorf("could not encode pin: %v", err) + } + + p.setDirty(ctx, true) + + // Store CID index + switch mode { + case ipfspinner.Recursive: + err = p.cidRIndex.Add(ctx, c.KeyString(), pp.Id) + case ipfspinner.Direct: + err = p.cidDIndex.Add(ctx, c.KeyString(), pp.Id) + default: + panic("pin mode must be recursive or direct") + } + if err != nil { + return "", fmt.Errorf("could not add pin cid index: %v", err) + } + + if name != "" { + // Store name index + err = p.nameIndex.Add(ctx, name, pp.Id) + if err != nil { + return "", fmt.Errorf("could not add pin name index: %v", err) + } + } + + // Store the pin. Pin must be stored after index for recovery to work. + err = p.dstore.Put(pp.dsKey(), pinData) + if err != nil { + if mode == ipfspinner.Recursive { + p.cidRIndex.Delete(ctx, c.KeyString(), pp.Id) + } else { + p.cidDIndex.Delete(ctx, c.KeyString(), pp.Id) + } + if name != "" { + p.nameIndex.Delete(ctx, name, pp.Id) + } + return "", err + } + + return pp.Id, nil +} + +func (p *pinner) removePin(ctx context.Context, pp *pin) error { + p.setDirty(ctx, true) + + // Remove pin from datastore. Pin must be removed before index for + // recovery to work. + err := p.dstore.Delete(pp.dsKey()) + if err != nil { + return err + } + // Remove cid index from datastore + if pp.Mode == ipfspinner.Recursive { + err = p.cidRIndex.Delete(ctx, pp.Cid.KeyString(), pp.Id) + } else { + err = p.cidDIndex.Delete(ctx, pp.Cid.KeyString(), pp.Id) + } + if err != nil { + return err + } + + if pp.Name != "" { + // Remove name index from datastore + err = p.nameIndex.Delete(ctx, pp.Name, pp.Id) + if err != nil { + return err + } + } + + return nil +} + +// Unpin a given key +func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { + cidKey := c.KeyString() + + p.lock.Lock() + defer p.lock.Unlock() + + // TODO: use Ls() to lookup pins when new pinning API available + /* + matchSpec := map[string][]string { + "cid": []string{c.String} + } + matches := p.Ls(matchSpec) + */ + has, err := p.cidRIndex.HasAny(ctx, cidKey) + if err != nil { + return err + } + + if has { + if !recursive { + return fmt.Errorf("%s is pinned recursively", c.String()) + } + } else { + has, err = p.cidDIndex.HasAny(ctx, cidKey) + if err != nil { + return err + } + if !has { + return ErrNotPinned + } + } + + _, err = p.removePinsForCid(ctx, c, ipfspinner.Any) + if err != nil { + return err + } + + return nil +} + +// IsPinned returns whether or not the given key is pinned +// and an explanation of why its pinned +func (p *pinner) IsPinned(ctx context.Context, c cid.Cid) (string, bool, error) { + p.lock.RLock() + defer p.lock.RUnlock() + return p.isPinnedWithType(ctx, c, ipfspinner.Any) +} + +// IsPinnedWithType returns whether or not the given cid is pinned with the +// given pin type, as well as returning the type of pin its pinned with. +func (p *pinner) IsPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (string, bool, error) { + p.lock.RLock() + defer p.lock.RUnlock() + return p.isPinnedWithType(ctx, c, mode) +} + +func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (string, bool, error) { + cidKey := c.KeyString() + switch mode { + case ipfspinner.Recursive: + has, err := p.cidRIndex.HasAny(ctx, cidKey) + if err != nil { + return "", false, err + } + if has { + return linkRecursive, true, nil + } + return "", false, nil + case ipfspinner.Direct: + has, err := p.cidDIndex.HasAny(ctx, cidKey) + if err != nil { + return "", false, err + } + if has { + return linkDirect, true, nil + } + return "", false, nil + case ipfspinner.Internal: + return "", false, nil + case ipfspinner.Indirect: + case ipfspinner.Any: + has, err := p.cidRIndex.HasAny(ctx, cidKey) + if err != nil { + return "", false, err + } + if has { + return linkRecursive, true, nil + } + has, err = p.cidDIndex.HasAny(ctx, cidKey) + if err != nil { + return "", false, err + } + if has { + return linkDirect, true, nil + } + default: + err := fmt.Errorf( + "invalid Pin Mode '%d', must be one of {%d, %d, %d, %d, %d}", + mode, ipfspinner.Direct, ipfspinner.Indirect, ipfspinner.Recursive, + ipfspinner.Internal, ipfspinner.Any) + return "", false, err + } + + // Default is Indirect + visitedSet := cid.NewSet() + + // No index for given CID, so search children of all recursive pinned CIDs + var has bool + var rc cid.Cid + var e error + err := p.cidRIndex.ForEach(ctx, "", func(key, value string) bool { + rc, e = cid.Cast([]byte(key)) + if e != nil { + return false + } + has, e = hasChild(ctx, p.dserv, rc, c, visitedSet.Visit) + if e != nil { + return false + } + if has { + return false + } + return true + }) + if err != nil { + return "", false, err + } + if e != nil { + return "", false, e + } + + if has { + return rc.String(), true, nil + } + + return "", false, nil +} + +// CheckIfPinned checks if a set of keys are pinned, more efficient than +// calling IsPinned for each key, returns the pinned status of cid(s) +// +// TODO: If a CID is pinned by multiple pins, should they all be reported? +func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinner.Pinned, error) { + pinned := make([]ipfspinner.Pinned, 0, len(cids)) + toCheck := cid.NewSet() + + p.lock.RLock() + defer p.lock.RUnlock() + + // First check for non-Indirect pins directly + for _, c := range cids { + cidKey := c.KeyString() + has, err := p.cidRIndex.HasAny(ctx, cidKey) + if err != nil { + return nil, err + } + if has { + pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Recursive}) + } else { + has, err = p.cidDIndex.HasAny(ctx, cidKey) + if err != nil { + return nil, err + } + if has { + pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Direct}) + } else { + toCheck.Add(c) + } + } + } + + // Now walk all recursive pins to check for indirect pins + var checkChildren func(cid.Cid, cid.Cid) error + checkChildren = func(rk, parentKey cid.Cid) error { + links, err := ipld.GetLinks(ctx, p.dserv, parentKey) + if err != nil { + return err + } + for _, lnk := range links { + c := lnk.Cid + + if toCheck.Has(c) { + pinned = append(pinned, + ipfspinner.Pinned{Key: c, Mode: ipfspinner.Indirect, Via: rk}) + toCheck.Remove(c) + } + + err = checkChildren(rk, c) + if err != nil { + return err + } + + if toCheck.Len() == 0 { + return nil + } + } + return nil + } + + var e error + err := p.cidRIndex.ForEach(ctx, "", func(key, value string) bool { + var rk cid.Cid + rk, e = cid.Cast([]byte(key)) + if e != nil { + return false + } + e = checkChildren(rk, rk) + if e != nil { + return false + } + if toCheck.Len() == 0 { + return false + } + return true + }) + if err != nil { + return nil, err + } + if e != nil { + return nil, e + } + + // Anything left in toCheck is not pinned + for _, k := range toCheck.Keys() { + pinned = append(pinned, ipfspinner.Pinned{Key: k, Mode: ipfspinner.NotPinned}) + } + + return pinned, nil +} + +// RemovePinWithMode is for manually editing the pin structure. +// Use with care! If used improperly, garbage collection may not +// be successful. +func (p *pinner) RemovePinWithMode(c cid.Cid, mode ipfspinner.Mode) { + ctx := context.TODO() + // Check cache to see if CID is pinned + switch mode { + case ipfspinner.Direct, ipfspinner.Recursive: + default: + // programmer error, panic OK + panic("unrecognized pin type") + } + + p.lock.Lock() + defer p.lock.Unlock() + + p.removePinsForCid(ctx, c, mode) +} + +// removePinsForCid removes all pins for a cid that has the specified mode. +// Returns true if any pins, and all corresponding CID index entries, were +// removed. Otherwise, returns false. +func (p *pinner) removePinsForCid(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (bool, error) { + // Search for pins by CID + var ids []string + var err error + cidKey := c.KeyString() + switch mode { + case ipfspinner.Recursive: + ids, err = p.cidRIndex.Search(ctx, cidKey) + case ipfspinner.Direct: + ids, err = p.cidDIndex.Search(ctx, cidKey) + case ipfspinner.Any: + ids, err = p.cidRIndex.Search(ctx, cidKey) + if err != nil { + return false, err + } + dIds, err := p.cidDIndex.Search(ctx, cidKey) + if err != nil { + return false, err + } + if len(dIds) != 0 { + ids = append(ids, dIds...) + } + } + if err != nil { + return false, err + } + + var removed bool + + // Remove the pin with the requested mode + for _, pid := range ids { + var pp *pin + pp, err = p.loadPin(ctx, pid) + if err != nil { + if err == ds.ErrNotFound { + p.setDirty(ctx, true) + // Fix index; remove index for pin that does not exist + switch mode { + case ipfspinner.Recursive: + p.cidRIndex.DeleteKey(ctx, cidKey) + case ipfspinner.Direct: + p.cidDIndex.DeleteKey(ctx, cidKey) + case ipfspinner.Any: + p.cidRIndex.DeleteKey(ctx, cidKey) + p.cidDIndex.DeleteKey(ctx, cidKey) + } + log.Error("found CID index with missing pin") + continue + } + return false, err + } + if mode == ipfspinner.Any || pp.Mode == mode { + err = p.removePin(ctx, pp) + if err != nil { + return false, err + } + removed = true + } + } + return removed, nil +} + +// loadPin loads a single pin from the datastore. +func (p *pinner) loadPin(ctx context.Context, pid string) (*pin, error) { + pinData, err := p.dstore.Get(ds.NewKey(path.Join(pinKeyPath, pid))) + if err != nil { + return nil, err + } + return decodePin(pid, pinData) +} + +// loadAllPins loads all pins from the datastore. +func (p *pinner) loadAllPins(ctx context.Context) ([]*pin, error) { + q := query.Query{ + Prefix: pinKeyPath, + } + results, err := p.dstore.Query(q) + if err != nil { + return nil, err + } + ents, err := results.Rest() + if err != nil { + return nil, err + } + if len(ents) == 0 { + return nil, nil + } + + pins := make([]*pin, len(ents)) + for i := range ents { + if ctx.Err() != nil { + return nil, ctx.Err() + } + var p *pin + p, err = decodePin(path.Base(ents[i].Key), ents[i].Value) + if err != nil { + return nil, err + } + pins[i] = p + } + return pins, nil +} + +// rebuildIndexes uses the stored pins to rebuild secondary indexes. This +// resolves any discrepancy between secondary indexes and pins that could +// result from a program termination between saving the two. +func (p *pinner) rebuildIndexes(ctx context.Context, pins []*pin) error { + // Build temporary in-memory CID index from pins + dstoreMem := ds.NewMapDatastore() + tmpCidDIndex := dsindex.New(dstoreMem, ds.NewKey(pinCidDIndexPath)) + tmpCidRIndex := dsindex.New(dstoreMem, ds.NewKey(pinCidRIndexPath)) + tmpNameIndex := dsindex.New(dstoreMem, ds.NewKey(pinNameIndexPath)) + var hasNames bool + for _, pp := range pins { + if ctx.Err() != nil { + return ctx.Err() + } + if pp.Mode == ipfspinner.Recursive { + tmpCidRIndex.Add(ctx, pp.Cid.KeyString(), pp.Id) + } else if pp.Mode == ipfspinner.Direct { + tmpCidDIndex.Add(ctx, pp.Cid.KeyString(), pp.Id) + } + if pp.Name != "" { + tmpNameIndex.Add(ctx, pp.Name, pp.Id) + hasNames = true + } + } + + // Sync the CID index to what was build from pins. This fixes any invalid + // indexes, which could happen if ipfs was terminated between writing pin + // and writing secondary index. + changed, err := dsindex.SyncIndex(ctx, tmpCidRIndex, p.cidRIndex) + if err != nil { + return fmt.Errorf("cannot sync indexes: %v", err) + } + if changed { + log.Info("invalid recursive indexes detected - rebuilt") + } + + changed, err = dsindex.SyncIndex(ctx, tmpCidDIndex, p.cidDIndex) + if err != nil { + return fmt.Errorf("cannot sync indexes: %v", err) + } + if changed { + log.Info("invalid direct indexes detected - rebuilt") + } + + if hasNames { + changed, err = dsindex.SyncIndex(ctx, tmpNameIndex, p.nameIndex) + if err != nil { + return fmt.Errorf("cannot sync name indexes: %v", err) + } + if changed { + log.Info("invalid name indexes detected - rebuilt") + } + } + + return p.Flush(ctx) +} + +// DirectKeys returns a slice containing the directly pinned keys +func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { + p.lock.RLock() + defer p.lock.RUnlock() + + cidSet := cid.NewSet() + var e error + err := p.cidDIndex.ForEach(ctx, "", func(key, value string) bool { + var c cid.Cid + c, e = cid.Cast([]byte(key)) + if e != nil { + return false + } + cidSet.Add(c) + return true + }) + if err != nil { + return nil, err + } + if e != nil { + return nil, e + } + + return cidSet.Keys(), nil +} + +// RecursiveKeys returns a slice containing the recursively pinned keys +func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { + p.lock.RLock() + defer p.lock.RUnlock() + + cidSet := cid.NewSet() + var e error + err := p.cidRIndex.ForEach(ctx, "", func(key, value string) bool { + var c cid.Cid + c, e = cid.Cast([]byte(key)) + if e != nil { + return false + } + cidSet.Add(c) + return true + }) + if err != nil { + return nil, err + } + if e != nil { + return nil, e + } + + return cidSet.Keys(), nil +} + +// InternalPins returns all cids kept pinned for the internal state of the +// pinner +func (p *pinner) InternalPins(ctx context.Context) ([]cid.Cid, error) { + return nil, nil +} + +// Update updates a recursive pin from one cid to another. This is equivalent +// to pinning the new one and unpinning the old one. +// +// TODO: This will not work when multiple pins are supported +func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error { + p.lock.Lock() + defer p.lock.Unlock() + + found, err := p.cidRIndex.HasAny(ctx, from.KeyString()) + if err != nil { + return err + } + if !found { + return errors.New("'from' cid was not recursively pinned already") + } + + // If `from` already recursively pinned and `to` is the same, then all done + if from == to { + return nil + } + + // Check if the `to` cid is already recursively pinned + found, err = p.cidRIndex.HasAny(ctx, to.KeyString()) + if err != nil { + return err + } + if found { + return errors.New("'to' cid was already recursively pinned") + } + + // Temporarily unlock while we fetch the differences. + p.lock.Unlock() + err = dagutils.DiffEnumerate(ctx, p.dserv, from, to) + p.lock.Lock() + + if err != nil { + return err + } + + _, err = p.addPin(ctx, to, ipfspinner.Recursive, "") + if err != nil { + return err + } + + if !unpin { + return nil + } + + _, err = p.removePinsForCid(ctx, from, ipfspinner.Recursive) + if err != nil { + return err + } + + return nil +} + +// Flush encodes and writes pinner keysets to the datastore +func (p *pinner) Flush(ctx context.Context) error { + p.lock.Lock() + defer p.lock.Unlock() + + if syncDServ, ok := p.dserv.(syncDAGService); ok { + if err := syncDServ.Sync(); err != nil { + return fmt.Errorf("cannot sync pinned data: %v", err) + } + } + + // Sync pins and indexes + if err := p.dstore.Sync(ds.NewKey(basePath)); err != nil { + return fmt.Errorf("cannot sync pin state: %v", err) + } + + p.setDirty(ctx, false) + + return nil +} + +// PinWithMode allows the user to have fine grained control over pin +// counts +func (p *pinner) PinWithMode(c cid.Cid, mode ipfspinner.Mode) { + ctx := context.TODO() + + p.lock.Lock() + defer p.lock.Unlock() + + // TODO: remove his to support multiple pins per CID + switch mode { + case ipfspinner.Recursive: + if has, _ := p.cidRIndex.HasAny(ctx, c.KeyString()); has { + return // already a recursive pin for this CID + } + case ipfspinner.Direct: + if has, _ := p.cidDIndex.HasAny(ctx, c.KeyString()); has { + return // already a direct pin for this CID + } + default: + panic("unrecognized pin mode") + } + + _, err := p.addPin(ctx, c, mode, "") + if err != nil { + return + } +} + +// hasChild recursively looks for a Cid among the children of a root Cid. +// The visit function can be used to shortcut already-visited branches. +func hasChild(ctx context.Context, ng ipld.NodeGetter, root cid.Cid, child cid.Cid, visit func(cid.Cid) bool) (bool, error) { + links, err := ipld.GetLinks(ctx, ng, root) + if err != nil { + return false, err + } + for _, lnk := range links { + c := lnk.Cid + if lnk.Cid.Equals(child) { + return true, nil + } + if visit(c) { + has, err := hasChild(ctx, ng, c, child, visit) + if err != nil { + return false, err + } + + if has { + return has, nil + } + } + } + return false, nil +} + +func encodePin(p *pin) ([]byte, error) { + b, err := cbor.MarshalAtlased(p, pinAtl) + if err != nil { + return nil, err + } + return b, nil +} + +func decodePin(pid string, data []byte) (*pin, error) { + p := &pin{Id: pid} + err := cbor.UnmarshalAtlased(cbor.DecodeOptions{}, data, p, pinAtl) + if err != nil { + return nil, err + } + return p, nil +} + +// setDirty saves a boolean dirty flag in the datastore whenever there is a +// transition between a dirty (counter > 0) and non-dirty (counter == 0) state. +func (p *pinner) setDirty(ctx context.Context, dirty bool) { + isClean := p.dirty == p.clean + if dirty { + p.dirty++ + if !isClean { + return // do not save; was already dirty + } + } else if isClean { + return // already clean + } else { + p.clean = p.dirty // set clean + } + + // Do edge-triggered write to datastore + data := []byte{0} + if dirty { + data[0] = 1 + } + p.dstore.Put(dirtyKey, data) + p.dstore.Sync(dirtyKey) +} diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go new file mode 100644 index 0000000..40e2c70 --- /dev/null +++ b/dspinner/pin_test.go @@ -0,0 +1,1137 @@ +package dspinner + +import ( + "context" + "errors" + "fmt" + "io" + "testing" + "time" + + bs "github.com/ipfs/go-blockservice" + mdag "github.com/ipfs/go-merkledag" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + lds "github.com/ipfs/go-ds-leveldb" + blockstore "github.com/ipfs/go-ipfs-blockstore" + offline "github.com/ipfs/go-ipfs-exchange-offline" + ipfspin "github.com/ipfs/go-ipfs-pinner" + "github.com/ipfs/go-ipfs-pinner/ipldpinner" + util "github.com/ipfs/go-ipfs-util" + ipld "github.com/ipfs/go-ipld-format" + logging "github.com/ipfs/go-log" +) + +var rand = util.NewTimeSeededRand() + +type fakeLogger struct { + logging.StandardLogger + lastError error +} + +func (f *fakeLogger) Error(args ...interface{}) { + f.lastError = errors.New(fmt.Sprint(args...)) +} + +func (f *fakeLogger) Errorf(format string, args ...interface{}) { + f.lastError = fmt.Errorf(format, args...) +} + +func randNode() (*mdag.ProtoNode, cid.Cid) { + nd := new(mdag.ProtoNode) + nd.SetData(make([]byte, 32)) + _, err := io.ReadFull(rand, nd.Data()) + if err != nil { + panic(err) + } + k := nd.Cid() + return nd, k +} + +func assertPinned(t *testing.T, p ipfspin.Pinner, c cid.Cid, failmsg string) { + _, pinned, err := p.IsPinned(context.Background(), c) + if err != nil { + t.Fatal(err) + } + + if !pinned { + t.Fatal(failmsg) + } +} + +func assertPinnedWithType(t *testing.T, p ipfspin.Pinner, c cid.Cid, mode ipfspin.Mode, failmsg string) { + modeText, pinned, err := p.IsPinnedWithType(context.Background(), c, mode) + if err != nil { + t.Fatal(err) + } + + expect, ok := ipfspin.ModeToString(mode) + if !ok { + t.Fatal("unrecognized pin mode") + } + + if !pinned { + t.Fatal(failmsg) + } + + if mode == ipfspin.Any { + return + } + + if expect != modeText { + t.Fatal("expected", expect, "pin, got", modeText) + } +} + +func assertUnpinned(t *testing.T, p ipfspin.Pinner, c cid.Cid, failmsg string) { + _, pinned, err := p.IsPinned(context.Background(), c) + if err != nil { + t.Fatal(err) + } + + if pinned { + t.Fatal(failmsg) + } +} + +func TestPinnerBasic(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + p, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + a, ak := randNode() + err = dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + + // Pin A{} + err = p.Pin(ctx, a, false) + if err != nil { + t.Fatal(err) + } + + assertPinned(t, p, ak, "Failed to find key") + assertPinnedWithType(t, p, ak, ipfspin.Direct, "Expected direct pin") + + // create new node c, to be indirectly pinned through b + c, _ := randNode() + err = dserv.Add(ctx, c) + if err != nil { + t.Fatal(err) + } + ck := c.Cid() + + // Create new node b, to be parent to a and c + b, _ := randNode() + err = b.AddNodeLink("child", a) + if err != nil { + t.Fatal(err) + } + err = b.AddNodeLink("otherchild", c) + if err != nil { + t.Fatal(err) + } + + err = dserv.Add(ctx, b) + if err != nil { + t.Fatal(err) + } + bk := b.Cid() + + // recursively pin B{A,C} + err = p.Pin(ctx, b, true) + if err != nil { + t.Fatal(err) + } + + assertPinned(t, p, ck, "child of recursively pinned node not found") + + assertPinned(t, p, bk, "Pinned node not found") + assertPinnedWithType(t, p, bk, ipfspin.Recursive, "Recursively pinned node not found") + + d, _ := randNode() + d.AddNodeLink("a", a) + d.AddNodeLink("c", c) + + e, _ := randNode() + d.AddNodeLink("e", e) + + // Must be in dagserv for unpin to work + err = dserv.Add(ctx, e) + if err != nil { + t.Fatal(err) + } + err = dserv.Add(ctx, d) + if err != nil { + t.Fatal(err) + } + + // Add D{A,C,E} + err = p.Pin(ctx, d, true) + if err != nil { + t.Fatal(err) + } + + dk := d.Cid() + assertPinned(t, p, dk, "pinned node not found.") + + cids, err := p.RecursiveKeys(ctx) + if err != nil { + t.Fatal(err) + } + if len(cids) != 2 { + t.Error("expected 2 recursive pins") + } + if !(bk == cids[0] || bk == cids[1]) { + t.Error("expected recursive pin of B") + } + if !(dk == cids[0] || dk == cids[1]) { + t.Error("expected recursive pin of D") + } + + pinned, err := p.CheckIfPinned(ctx, ak, bk, ck, dk) + if err != nil { + t.Fatal(err) + } + if len(pinned) != 4 { + t.Error("incorrect number of results") + } + for _, pn := range pinned { + switch pn.Key { + case ak: + if pn.Mode != ipfspin.Direct { + t.Error("A pinned with wrong mode") + } + case bk: + if pn.Mode != ipfspin.Recursive { + t.Error("B pinned with wrong mode") + } + case ck: + if pn.Mode != ipfspin.Indirect { + t.Error("C should be pinned indirectly") + } + if pn.Via != dk && pn.Via != bk { + t.Error("C should be pinned via D or B") + } + case dk: + if pn.Mode != ipfspin.Recursive { + t.Error("D pinned with wrong mode") + } + } + } + + cids, err = p.DirectKeys(ctx) + if err != nil { + t.Fatal(err) + } + if len(cids) != 1 { + t.Error("expected 1 direct pin") + } + if cids[0] != ak { + t.Error("wrong direct pin") + } + + cids, _ = p.InternalPins(ctx) + if len(cids) != 0 { + t.Error("shound not have internal keys") + } + + err = p.Unpin(ctx, dk, false) + if err == nil { + t.Fatal("expected error unpinning recursive pin without specifying recursive") + } + + // Test recursive unpin + err = p.Unpin(ctx, dk, true) + if err != nil { + t.Fatal(err) + } + + err = p.Unpin(ctx, dk, true) + if err != ErrNotPinned { + t.Fatal("expected error:", ErrNotPinned) + } + + err = p.Flush(ctx) + if err != nil { + t.Fatal(err) + } + + p, err = New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + // Test directly pinned + assertPinned(t, p, ak, "Could not find pinned node!") + + // Test recursively pinned + assertPinned(t, p, bk, "could not find recursively pinned node") + + // Remove the pin but not the index to simulate corruption + dsp := p.(*pinner) + ids, err := dsp.cidDIndex.Search(ctx, ak.KeyString()) + if err != nil { + t.Fatal(err) + } + if len(ids) == 0 { + t.Fatal("did not find pin for cid", ak.String()) + } + pp, err := dsp.loadPin(ctx, ids[0]) + if err != nil { + t.Fatal(err) + } + if pp.Mode != ipfspin.Direct { + t.Error("loaded pin has wrong mode") + } + if pp.Cid != ak { + t.Error("loaded pin has wrong cid") + } + err = dsp.dstore.Delete(pp.dsKey()) + if err != nil { + t.Fatal(err) + } + + realLog := log + fakeLog := &fakeLogger{} + fakeLog.StandardLogger = log + log = fakeLog + err = p.Pin(ctx, a, true) + if err != nil { + t.Fatal(err) + } + if fakeLog.lastError == nil { + t.Error("expected error to be logged") + } else if fakeLog.lastError.Error() != "found CID index with missing pin" { + t.Error("did not get expected log message") + } + + log = realLog +} + +func TestAddLoadPin(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + ipfsPin, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + p := ipfsPin.(*pinner) + + a, ak := randNode() + dserv.Add(ctx, a) + + mode := ipfspin.Recursive + name := "my-pin" + pid, err := p.addPin(ctx, ak, mode, name) + if err != nil { + t.Fatal(err) + } + + // Load pin and check that data decoded correctly + pinData, err := p.loadPin(ctx, pid) + if err != nil { + t.Fatal(err) + } + if pinData.Mode != mode { + t.Error("worng pin mode") + } + if pinData.Cid != ak { + t.Error("wrong pin cid") + } + if pinData.Name != name { + t.Error("wrong pin name; expected", name, "got", pinData.Name) + } +} + +func TestRemovePinWithMode(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + p, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + a, ak := randNode() + dserv.Add(ctx, a) + + p.Pin(ctx, a, false) + + ok, err := p.(*pinner).removePinsForCid(ctx, ak, ipfspin.Recursive) + if err != nil { + t.Fatal(err) + } + if ok { + t.Error("pin should not have been removed") + } + + p.RemovePinWithMode(ak, ipfspin.Direct) + + assertUnpinned(t, p, ak, "pin was not removed") +} + +func TestIsPinnedLookup(t *testing.T) { + // Test that lookups work in pins which share + // the same branches. For that construct this tree: + // + // A5->A4->A3->A2->A1->A0 + // / / + // B------- / + // \ / + // C--------------- + // + // This ensures that IsPinned works for all objects both when they + // are pinned and once they have been unpinned. + aBranchLen := 6 + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + // Create new pinner. New will not load anything since there are + // no pins saved in the datastore yet. + p, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + aKeys, bk, ck, err := makeTree(ctx, aBranchLen, dserv, p) + if err != nil { + t.Fatal(err) + } + + assertPinned(t, p, aKeys[0], "A0 should be pinned") + assertPinned(t, p, aKeys[1], "A1 should be pinned") + assertPinned(t, p, ck, "C should be pinned") + assertPinned(t, p, bk, "B should be pinned") + + // Unpin A5 recursively + if err = p.Unpin(ctx, aKeys[5], true); err != nil { + t.Fatal(err) + } + + assertPinned(t, p, aKeys[0], "A0 should still be pinned through B") + assertUnpinned(t, p, aKeys[4], "A4 should be unpinned") + + // Unpin B recursively + if err = p.Unpin(ctx, bk, true); err != nil { + t.Fatal(err) + } + assertUnpinned(t, p, bk, "B should be unpinned") + assertUnpinned(t, p, aKeys[1], "A1 should be unpinned") + assertPinned(t, p, aKeys[0], "A0 should still be pinned through C") +} + +func TestDuplicateSemantics(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + p, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + a, _ := randNode() + err = dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + + // pin is recursively + err = p.Pin(ctx, a, true) + if err != nil { + t.Fatal(err) + } + + // pinning directly should fail + err = p.Pin(ctx, a, false) + if err == nil { + t.Fatal("expected direct pin to fail") + } + + // pinning recursively again should succeed + err = p.Pin(ctx, a, true) + if err != nil { + t.Fatal(err) + } +} + +func TestFlush(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + p, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + _, k := randNode() + + p.PinWithMode(k, ipfspin.Recursive) + if err = p.Flush(ctx); err != nil { + t.Fatal(err) + } + assertPinned(t, p, k, "expected key to still be pinned") +} + +func TestPinRecursiveFail(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + dserv := mdag.NewDAGService(bserv) + + p, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + a, _ := randNode() + b, _ := randNode() + err = a.AddNodeLink("child", b) + if err != nil { + t.Fatal(err) + } + + // NOTE: This isnt a time based test, we expect the pin to fail + mctx, cancel := context.WithTimeout(ctx, time.Millisecond) + defer cancel() + + err = p.Pin(mctx, a, true) + if err == nil { + t.Fatal("should have failed to pin here") + } + + err = dserv.Add(ctx, b) + if err != nil { + t.Fatal(err) + } + + err = dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + + // this one is time based... but shouldnt cause any issues + mctx, cancel = context.WithTimeout(ctx, time.Second) + defer cancel() + err = p.Pin(mctx, a, true) + if err != nil { + t.Fatal(err) + } +} + +func TestPinUpdate(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + p, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + n1, c1 := randNode() + n2, c2 := randNode() + _, c3 := randNode() + + if err = dserv.Add(ctx, n1); err != nil { + t.Fatal(err) + } + if err = dserv.Add(ctx, n2); err != nil { + t.Fatal(err) + } + + if err = p.Pin(ctx, n1, true); err != nil { + t.Fatal(err) + } + + if err = p.Update(ctx, c1, c2, true); err != nil { + t.Fatal(err) + } + + assertPinned(t, p, c2, "c2 should be pinned now") + assertUnpinned(t, p, c1, "c1 should no longer be pinned") + + if err = p.Update(ctx, c2, c1, false); err != nil { + t.Fatal(err) + } + + // Test updating same pin that is already pinned. + if err = p.Update(ctx, c2, c2, true); err != nil { + t.Fatal(err) + } + // Check that pin is still pinned. + _, ok, err := p.IsPinned(ctx, c2) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("c2 should still be pinned") + } + + // Test updating same pin that is not pinned. + if err = p.Update(ctx, c3, c3, false); err == nil { + t.Fatal("expected error updating unpinned cid") + } + _, ok, err = p.IsPinned(ctx, c3) + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatal("c3 should not be pinned") + } + + assertPinned(t, p, c2, "c2 should be pinned still") + assertPinned(t, p, c1, "c1 should be pinned now") +} + +func TestLoadDirty(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + dserv := mdag.NewDAGService(bserv) + + p, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + a, ak := randNode() + err = dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + + _, bk := randNode() + + err = p.Pin(ctx, a, true) + if err != nil { + t.Fatal(err) + } + + cidAKey := ak.KeyString() + cidBKey := bk.KeyString() + + // Corrupt index + cidRIndex := p.(*pinner).cidRIndex + cidRIndex.DeleteKey(ctx, cidAKey) + cidRIndex.Add(ctx, cidBKey, "not-a-pin-id") + + // Verify dirty + data, err := dstore.Get(dirtyKey) + if err != nil { + t.Fatalf("could not read dirty flag: %v", err) + } + if data[0] != 1 { + t.Fatal("dirty flag not set") + } + + has, err := cidRIndex.HasAny(ctx, cidAKey) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatal("index should be deleted") + } + + // Create new pinner on same datastore that was never flushed. + p, err = New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + // Verify not dirty + data, err = dstore.Get(dirtyKey) + if err != nil { + t.Fatalf("could not read dirty flag: %v", err) + } + if data[0] != 0 { + t.Fatal("dirty flag is set") + } + + // Verify index rebuilt + cidRIndex = p.(*pinner).cidRIndex + has, err = cidRIndex.HasAny(ctx, cidAKey) + if err != nil { + t.Fatal(err) + } + if !has { + t.Fatal("index should have been rebuilt") + } + + has, err = cidRIndex.HasAny(ctx, cidBKey) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatal("index should have been removed by rebuild") + } +} + +func TestEncodeDecodePin(t *testing.T) { + _, c := randNode() + + pin := newPin(c, ipfspin.Recursive, "testpin") + pin.Metadata = make(map[string]interface{}, 2) + pin.Metadata["hello"] = "world" + pin.Metadata["foo"] = "bar" + + encBytes, err := encodePin(pin) + if err != nil { + t.Fatal(err) + } + + decPin, err := decodePin(pin.Id, encBytes) + if err != nil { + t.Fatal(err) + } + + if decPin.Id != pin.Id { + t.Errorf("wrong pin id: expect %q got %q", pin.Id, decPin.Id) + } + if decPin.Cid != pin.Cid { + t.Errorf("wrong pin cid: expect %q got %q", pin.Cid.String(), decPin.Cid.String()) + } + if decPin.Mode != pin.Mode { + expect, _ := ipfspin.ModeToString(pin.Mode) + got, _ := ipfspin.ModeToString(decPin.Mode) + t.Errorf("wrong pin mode: expect %s got %s", expect, got) + } + if decPin.Name != pin.Name { + t.Errorf("wrong pin name: expect %q got %q", pin.Name, decPin.Name) + } + for key, val := range pin.Metadata { + dval, ok := decPin.Metadata[key] + if !ok { + t.Errorf("decoded pin missing metadata key %q", key) + } + if dval != val { + t.Errorf("wrong metadata value: expected %q got %q", val, dval) + } + } +} + +func makeTree(ctx context.Context, aBranchLen int, dserv ipld.DAGService, p ipfspin.Pinner) (aKeys []cid.Cid, bk cid.Cid, ck cid.Cid, err error) { + if aBranchLen < 3 { + err = errors.New("set aBranchLen to at least 3") + return + } + + aNodes := make([]*mdag.ProtoNode, aBranchLen) + aKeys = make([]cid.Cid, aBranchLen) + for i := 0; i < aBranchLen; i++ { + a, _ := randNode() + if i >= 1 { + if err = a.AddNodeLink("child", aNodes[i-1]); err != nil { + return + } + } + + if err = dserv.Add(ctx, a); err != nil { + return + } + aNodes[i] = a + aKeys[i] = a.Cid() + } + + // Pin last A recursively + if err = p.Pin(ctx, aNodes[aBranchLen-1], true); err != nil { + return + } + + // Create node B and add A3 as child + b, _ := randNode() + if err = b.AddNodeLink("mychild", aNodes[3]); err != nil { + return + } + + // Create C node + c, _ := randNode() + // Add A0 as child of C + if err = c.AddNodeLink("child", aNodes[0]); err != nil { + return + } + + // Add C + if err = dserv.Add(ctx, c); err != nil { + return + } + ck = c.Cid() + + // Add C to B and Add B + if err = b.AddNodeLink("myotherchild", c); err != nil { + return + } + if err = dserv.Add(ctx, b); err != nil { + return + } + bk = b.Cid() + + // Pin C recursively + if err = p.Pin(ctx, c, true); err != nil { + return + } + + // Pin B recursively + if err = p.Pin(ctx, b, true); err != nil { + return + } + + if err = p.Flush(ctx); err != nil { + return + } + + return +} + +func makeNodes(count int, dserv ipld.DAGService) []ipld.Node { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nodes := make([]ipld.Node, count) + for i := 0; i < count; i++ { + n, _ := randNode() + err := dserv.Add(ctx, n) + if err != nil { + panic(err) + } + nodes[i] = n + } + return nodes +} + +func pinNodes(nodes []ipld.Node, p ipfspin.Pinner, recursive bool) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var err error + + for i := range nodes { + err = p.Pin(ctx, nodes[i], recursive) + if err != nil { + panic(err) + } + } + err = p.Flush(ctx) + if err != nil { + panic(err) + } +} + +func unpinNodes(nodes []ipld.Node, p ipfspin.Pinner) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var err error + + for i := range nodes { + err = p.Unpin(ctx, nodes[i].Cid(), true) + if err != nil { + panic(err) + } + } + err = p.Flush(ctx) + if err != nil { + panic(err) + } +} + +type batchWrap struct { + ds.Datastore +} + +func (d *batchWrap) Batch() (ds.Batch, error) { + return ds.NewBasicBatch(d), nil +} + +func makeStore() (ds.Datastore, ipld.DAGService) { + ldstore, err := lds.NewDatastore("", nil) + if err != nil { + panic(err) + } + var dstore ds.Batching + dstore = &batchWrap{ldstore} + + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + dserv := mdag.NewDAGService(bserv) + return dstore, dserv +} + +// BenchmarkLoadRebuild loads a pinner that has some number of saved pins, and +// compares the load time when rebuilding indexes to loading without rebuilding +// indexes. +func BenchmarkLoadRebuild(b *testing.B) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dstore, dserv := makeStore() + pinner, err := New(ctx, dstore, dserv) + if err != nil { + panic(err.Error()) + } + + nodes := makeNodes(4096, dserv) + pinNodes(nodes, pinner, true) + + b.Run("RebuildTrue", func(b *testing.B) { + for i := 0; i < b.N; i++ { + dstore.Put(dirtyKey, []byte{1}) + + _, err = New(ctx, dstore, dserv) + if err != nil { + panic(err.Error()) + } + } + }) + + b.Run("RebuildFalse", func(b *testing.B) { + for i := 0; i < b.N; i++ { + dstore.Put(dirtyKey, []byte{0}) + + _, err = New(ctx, dstore, dserv) + if err != nil { + panic(err.Error()) + } + } + }) +} + +// BenchmarkNthPins shows the time it takes to create/save 1 pin when a number +// of other pins already exist. Each run in the series shows performance for +// creating a pin in a larger number of existing pins. +func BenchmarkNthPin(b *testing.B) { + dstore, dserv := makeStore() + pinner, err := New(context.Background(), dstore, dserv) + if err != nil { + panic(err.Error()) + } + pinnerIPLD, err := ipldpinner.New(dstore, dserv, dserv) + if err != nil { + panic(err.Error()) + } + + for count := 1000; count <= 10000; count += 1000 { + b.Run(fmt.Sprint("PinDS-", count), func(b *testing.B) { + benchmarkNthPin(b, count, pinner, dserv) + }) + + b.Run(fmt.Sprint("PinIPLD-", count), func(b *testing.B) { + benchmarkNthPin(b, count, pinnerIPLD, dserv) + }) + } +} + +func benchmarkNthPin(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nodes := makeNodes(count, dserv) + pinNodes(nodes[:count-1], pinner, true) + b.ResetTimer() + + which := count - 1 + for i := 0; i < b.N; i++ { + // Pin the Nth node and Flush + err := pinner.Pin(ctx, nodes[which], true) + if err != nil { + panic(err) + } + err = pinner.Flush(ctx) + if err != nil { + panic(err) + } + // Unpin the nodes so that it can pinned next iter. + b.StopTimer() + err = pinner.Unpin(ctx, nodes[which].Cid(), true) + if err != nil { + panic(err) + } + err = pinner.Flush(ctx) + if err != nil { + panic(err) + } + b.StartTimer() + } +} + +// BenchmarkNPins demonstrates creating individual pins. Each run in the +// series shows performance for a larger number of individual pins. +func BenchmarkNPins(b *testing.B) { + for count := 128; count < 16386; count <<= 1 { + b.Run(fmt.Sprint("PinDS-", count), func(b *testing.B) { + dstore, dserv := makeStore() + pinner, err := New(context.Background(), dstore, dserv) + if err != nil { + panic(err.Error()) + } + benchmarkNPins(b, count, pinner, dserv) + }) + + b.Run(fmt.Sprint("PinIPLD-", count), func(b *testing.B) { + dstore, dserv := makeStore() + pinner, err := ipldpinner.New(dstore, dserv, dserv) + if err != nil { + panic(err.Error()) + } + benchmarkNPins(b, count, pinner, dserv) + }) + } +} + +func benchmarkNPins(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nodes := makeNodes(count, dserv) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + // Pin all the nodes one at a time. + for j := range nodes { + err := pinner.Pin(ctx, nodes[j], true) + if err != nil { + panic(err) + } + err = pinner.Flush(ctx) + if err != nil { + panic(err) + } + } + + // Unpin all nodes so that they can be pinned next iter. + b.StopTimer() + unpinNodes(nodes, pinner) + b.StartTimer() + } +} + +// BenchmarkNUnpins demonstrates unpinning individual pins. Each run in the +// series shows performance for a larger number of individual unpins. +func BenchmarkNUnpins(b *testing.B) { + for count := 128; count < 16386; count <<= 1 { + b.Run(fmt.Sprint("UnpinDS-", count), func(b *testing.B) { + dstore, dserv := makeStore() + pinner, err := New(context.Background(), dstore, dserv) + if err != nil { + panic(err.Error()) + } + benchmarkNUnpins(b, count, pinner, dserv) + }) + + b.Run(fmt.Sprint("UninIPLD-", count), func(b *testing.B) { + dstore, dserv := makeStore() + pinner, err := ipldpinner.New(dstore, dserv, dserv) + if err != nil { + panic(err.Error()) + } + benchmarkNUnpins(b, count, pinner, dserv) + }) + } +} + +func benchmarkNUnpins(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nodes := makeNodes(count, dserv) + pinNodes(nodes, pinner, true) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for j := range nodes { + // Unpin nodes one at a time. + err := pinner.Unpin(ctx, nodes[j].Cid(), true) + if err != nil { + panic(err) + } + err = pinner.Flush(ctx) + if err != nil { + panic(err) + } + } + // Pin all nodes so that they can be unpinned next iter. + b.StopTimer() + pinNodes(nodes, pinner, true) + b.StartTimer() + } +} + +// BenchmarkPinAllSeries shows times to pin all nodes with only one Flush at +// the end. +func BenchmarkPinAll(b *testing.B) { + for count := 128; count < 16386; count <<= 1 { + b.Run(fmt.Sprint("PinAllDS-", count), func(b *testing.B) { + dstore, dserv := makeStore() + pinner, err := New(context.Background(), dstore, dserv) + if err != nil { + panic(err) + } + benchmarkPinAll(b, count, pinner, dserv) + }) + + b.Run(fmt.Sprint("PinAllIPLD-", count), func(b *testing.B) { + dstore, dserv := makeStore() + pinner, err := ipldpinner.New(dstore, dserv, dserv) + if err != nil { + panic(err.Error()) + } + benchmarkPinAll(b, count, pinner, dserv) + }) + } +} + +func benchmarkPinAll(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { + nodes := makeNodes(count, dserv) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + pinNodes(nodes, pinner, true) + + b.StopTimer() + unpinNodes(nodes, pinner) + b.StartTimer() + } +} diff --git a/go.mod b/go.mod index 48479ae..5fb384a 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,16 @@ go 1.13 require ( github.com/gogo/protobuf v1.3.1 - github.com/ipfs/go-blockservice v0.1.2 - github.com/ipfs/go-cid v0.0.3 - github.com/ipfs/go-datastore v0.3.0 - github.com/ipfs/go-ipfs-blockstore v0.1.0 + github.com/ipfs/go-blockservice v0.1.4 + github.com/ipfs/go-cid v0.0.7 + github.com/ipfs/go-datastore v0.4.5 + github.com/ipfs/go-ds-leveldb v0.4.2 + github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 - github.com/ipfs/go-ipfs-util v0.0.1 - github.com/ipfs/go-ipld-format v0.0.2 - github.com/ipfs/go-log v0.0.1 + github.com/ipfs/go-ipfs-util v0.0.2 + github.com/ipfs/go-ipld-format v0.2.0 + github.com/ipfs/go-log v1.0.4 github.com/ipfs/go-merkledag v0.3.0 + github.com/multiformats/go-multibase v0.0.3 + github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 ) diff --git a/go.sum b/go.sum index c6a12e4..fc7de89 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMGXfDk= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -40,6 +42,7 @@ github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -49,6 +52,8 @@ github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfm github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= @@ -59,33 +64,40 @@ github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0r github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= -github.com/ipfs/go-bitswap v0.1.3 h1:jAl9Z/TYObpGeGATUemnOZ7RYb0F/kzNVlhcYZesz+0= -github.com/ipfs/go-bitswap v0.1.3/go.mod h1:YEQlFy0kkxops5Vy+OxWdRSEZIoS7I7KDIwoa5Chkps= +github.com/ipfs/go-bitswap v0.1.8 h1:38X1mKXkiU6Nzw4TOSWD8eTVY5eX3slQunv3QEWfXKg= +github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM= github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-blockservice v0.1.0 h1:dh2i7xjMbCtf0ZSMyQAF2qpV/pEEmM7yVpQ00+gik6U= github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= -github.com/ipfs/go-blockservice v0.1.2 h1:fqFeeu1EG0lGVrqUo+BVJv7LZV31I4ZsyNthCOMAJRc= -github.com/ipfs/go-blockservice v0.1.2/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= +github.com/ipfs/go-blockservice v0.1.4 h1:Vq+MlsH8000KbbUciRyYMEw/NNP8UAGmcqKi4uWmFGA= +github.com/ipfs/go-blockservice v0.1.4/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= -github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= -github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= +github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= +github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= +github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= -github.com/ipfs/go-datastore v0.1.0 h1:TOxI04l8CmO4zGtesENhzm4PwkFwJXY3rKiYaaMf9fI= -github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= -github.com/ipfs/go-datastore v0.3.0 h1:9au0tYi/+n7xeUnGHG6davnS8x9hWbOzP/388Vx3CMs= -github.com/ipfs/go-datastore v0.3.0/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= +github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= +github.com/ipfs/go-datastore v0.4.1 h1:W4ZfzyhNi3xmuU5dQhjfuRn/wFuqEE1KnOmmQiOevEY= +github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.5 h1:cwOUcGMLdLPWgu3SlrCckCMznaGADbPqE0r8h768/Dg= +github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= +github.com/ipfs/go-ds-leveldb v0.0.1 h1:Z0lsTFciec9qYsyngAw1f/czhRU35qBLR2vhavPFgqA= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= +github.com/ipfs/go-ds-leveldb v0.4.2 h1:QmQoAJ9WkPMUfBLnu1sBVy0xWWlJPg0m4kRAiJL9iaw= +github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xLX+8dPHzc= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= -github.com/ipfs/go-ipfs-blockstore v0.1.0 h1:V1GZorHFUIB6YgTJQdq7mcaIpUfCM3fCyVi+MTo9O88= -github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= +github.com/ipfs/go-ipfs-blockstore v0.1.4 h1:2SGI6U1B44aODevza8Rde3+dY30Pb+lbcObe1LETxOQ= +github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -93,6 +105,8 @@ github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1I github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v0.0.1 h1:QBg+Ts2zgeemK/dB0saiF/ykzRGgfoFMT90Rzo0OnVU= github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo= +github.com/ipfs/go-ipfs-ds-help v0.1.1 h1:IW/bXGeaAZV2VH0Kuok+Ohva/zHkHmeLFBxC1k7mNPc= +github.com/ipfs/go-ipfs-ds-help v0.1.1/go.mod h1:SbBafGJuGsPI/QL3j9Fc5YPLeAu+SzOkI0gFwAg+mOs= github.com/ipfs/go-ipfs-exchange-interface v0.0.1 h1:LJXIo9W7CAmugqI+uofioIpRb6rY30GUu7G6LUfpMvM= github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM= github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C81I1NSHW1FxGew= @@ -103,13 +117,21 @@ github.com/ipfs/go-ipfs-routing v0.1.0 h1:gAJTT1cEeeLj6/DlLX6t+NxD9fQe2ymTO6qWRD github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= +github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= +github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= github.com/ipfs/go-ipld-cbor v0.0.2 h1:amzFztBQQQ69UA5+f7JRfoXF/z2l//MGfEDHVkS20+s= github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.0.2 h1:OVAGlyYT6JPZ0pEfGntFPS40lfrDmaDbQwNHEY2G9Zs= github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= +github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA= +github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= +github.com/ipfs/go-log v1.0.4 h1:6nLQdX4W8P9yZZFH7mO+X/PzjN8Laozm/lMJ6esdgzY= +github.com/ipfs/go-log v1.0.4/go.mod h1:oDCg2FkjogeFOhqqb+N39l2RpTNPL6F/StPkB3kPgcs= +github.com/ipfs/go-log/v2 v2.0.5 h1:fL4YI+1g5V/b1Yxr1qAiXTMg1H8z9vx/VmJxBuQMHvU= +github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= github.com/ipfs/go-merkledag v0.3.0 h1:1bXv/ZRPZLVdij/a33CkXMVdxUdred9sz4xyph+0ls0= github.com/ipfs/go-merkledag v0.3.0/go.mod h1:4pymaZLhSLNVuiCITYrpViD6vmfZ/Ws4n/L9tfNv3S4= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= @@ -131,6 +153,8 @@ github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= @@ -143,6 +167,8 @@ github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b h1:wxtKgYHEncAU00muM github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -205,8 +231,8 @@ github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTW github.com/libp2p/go-mplex v0.1.0 h1:/nBTy5+1yRyY82YaO6HXQRnO5IAGsXTjEJaR3LdTPc0= github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU= github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= -github.com/libp2p/go-msgio v0.0.3 h1:VsOlWispTivSsOMg70e0W77y6oiSBSRCyP6URrWvE04= -github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= +github.com/libp2p/go-msgio v0.0.4 h1:agEFehY3zWJFUHK6SEMR7UYmk2z6kC3oeCM7ybLhguA= +github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-nat v0.0.3 h1:l6fKV+p0Xa354EqQOQP+d8CivdLM4kl5GxC1hSc/UeI= github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI= github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FWpFLw= @@ -238,12 +264,18 @@ github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.0 h1:U41/2erhAKcmSI14xh/ZTUdBPOzDOIfS93ibzUSl8KM= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.4 h1:WgMSI84/eRLdbptXMkMWDXPjPq7SPLIgGUVm2eroyU4= @@ -257,11 +289,18 @@ github.com/multiformats/go-multiaddr-net v0.0.1 h1:76O59E3FavvHqNg7jvzWzsPSW5JSi github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU= github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= +github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5 h1:1wxmCvTXAifAepIMyF39vZinRw5sbqjPs/UIi93+uik= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= +github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= +github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= @@ -278,6 +317,9 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 h1:bzMe+2coZJYHnhGgVlcQKuRy4FSny4ds8dLQjw5P1XE= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 h1:CskT+S6Ay54OwxBGB0R3Rsx4Muto6UnEYTyKJbyRIAI= +github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa h1:E+gaaifzi2xF65PbDmuKI3PhLWY6G5opMLniFq8vmXA= @@ -291,6 +333,8 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436 h1:qOpVTI+BrstcjTZLm2Yz/3sOnqkzj3FQoh0g+E5s3Gc= @@ -306,24 +350,40 @@ github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvX github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo= +go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a h1:+KkCgOMgnKSgenxTBoiwkMqTiouMIy/3o8RLdmSbGoY= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -340,12 +400,20 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -353,3 +421,5 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/ipldpinner/pin.go b/ipldpinner/pin.go new file mode 100644 index 0000000..d0824b3 --- /dev/null +++ b/ipldpinner/pin.go @@ -0,0 +1,528 @@ +// Package ipldpinner implements structures and methods to keep track of +// which objects a user wants to keep stored locally. This implementation +// stores pin information in a mdag structure. +package ipldpinner + +import ( + "context" + "fmt" + "os" + "sync" + "time" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + ipld "github.com/ipfs/go-ipld-format" + logging "github.com/ipfs/go-log" + mdag "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-merkledag/dagutils" + + ipfspinner "github.com/ipfs/go-ipfs-pinner" +) + +const loadTimeout = 5 * time.Second + +var log = logging.Logger("pin") + +var pinDatastoreKey = ds.NewKey("/local/pins") + +var emptyKey cid.Cid + +var linkDirect, linkRecursive, linkInternal string + +func init() { + e, err := cid.Decode("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n") + if err != nil { + log.Error("failed to decode empty key constant") + os.Exit(1) + } + emptyKey = e + + directStr, ok := ipfspinner.ModeToString(ipfspinner.Direct) + if !ok { + panic("could not find Direct pin enum") + } + linkDirect = directStr + + recursiveStr, ok := ipfspinner.ModeToString(ipfspinner.Recursive) + if !ok { + panic("could not find Recursive pin enum") + } + linkRecursive = recursiveStr + + internalStr, ok := ipfspinner.ModeToString(ipfspinner.Internal) + if !ok { + panic("could not find Internal pin enum") + } + linkInternal = internalStr +} + +// pinner implements the Pinner interface +type pinner struct { + lock sync.RWMutex + recursePin *cid.Set + directPin *cid.Set + + // Track the keys used for storing the pinning state, so gc does + // not delete them. + internalPin *cid.Set + dserv ipld.DAGService + internal ipld.DAGService // dagservice used to store internal objects + dstore ds.Datastore +} + +var _ ipfspinner.Pinner = (*pinner)(nil) + +type syncDAGService interface { + ipld.DAGService + Sync() error +} + +// New creates a new pinner using the given datastore as a backend, and loads +// the pinner's keysets from the datastore +func New(dstore ds.Datastore, dserv, internal ipld.DAGService) (*pinner, error) { + rootKey, err := dstore.Get(pinDatastoreKey) + if err != nil { + if err == ds.ErrNotFound { + return &pinner{ + recursePin: cid.NewSet(), + directPin: cid.NewSet(), + internalPin: cid.NewSet(), + dserv: dserv, + internal: internal, + dstore: dstore, + }, nil + } + return nil, err + } + rootCid, err := cid.Cast(rootKey) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(context.TODO(), loadTimeout) + defer cancel() + + root, err := internal.Get(ctx, rootCid) + if err != nil { + return nil, fmt.Errorf("cannot find pinning root object: %v", err) + } + + rootpb, ok := root.(*mdag.ProtoNode) + if !ok { + return nil, mdag.ErrNotProtobuf + } + + internalset := cid.NewSet() + internalset.Add(rootCid) + recordInternal := internalset.Add + + // load recursive set + recurseKeys, err := loadSet(ctx, internal, rootpb, linkRecursive, recordInternal) + if err != nil { + return nil, fmt.Errorf("cannot load recursive pins: %v", err) + } + + // load direct set + directKeys, err := loadSet(ctx, internal, rootpb, linkDirect, recordInternal) + if err != nil { + return nil, fmt.Errorf("cannot load direct pins: %v", err) + } + + return &pinner{ + // assign pinsets + recursePin: cidSetWithValues(recurseKeys), + directPin: cidSetWithValues(directKeys), + internalPin: internalset, + // assign services + dserv: dserv, + dstore: dstore, + internal: internal, + }, nil +} + +// Pin the given node, optionally recursive +func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { + err := p.dserv.Add(ctx, node) + if err != nil { + return err + } + + c := node.Cid() + + p.lock.Lock() + defer p.lock.Unlock() + + if recurse { + if p.recursePin.Has(c) { + return nil + } + + p.lock.Unlock() + // temporary unlock to fetch the entire graph + err := mdag.FetchGraph(ctx, c, p.dserv) + p.lock.Lock() + if err != nil { + return err + } + + if p.recursePin.Has(c) { + return nil + } + + if p.directPin.Has(c) { + p.directPin.Remove(c) + } + + p.recursePin.Add(c) + } else { + if p.recursePin.Has(c) { + return fmt.Errorf("%s already pinned recursively", c.String()) + } + + p.directPin.Add(c) + } + return nil +} + +// ErrNotPinned is returned when trying to unpin items which are not pinned. +var ErrNotPinned = fmt.Errorf("not pinned or pinned indirectly") + +// Unpin a given key +func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { + p.lock.Lock() + defer p.lock.Unlock() + if p.recursePin.Has(c) { + if !recursive { + return fmt.Errorf("%s is pinned recursively", c) + } + p.recursePin.Remove(c) + return nil + } + if p.directPin.Has(c) { + p.directPin.Remove(c) + return nil + } + return ErrNotPinned +} + +func (p *pinner) isInternalPin(c cid.Cid) bool { + return p.internalPin.Has(c) +} + +// IsPinned returns whether or not the given key is pinned +// and an explanation of why its pinned +func (p *pinner) IsPinned(ctx context.Context, c cid.Cid) (string, bool, error) { + p.lock.RLock() + defer p.lock.RUnlock() + return p.isPinnedWithType(ctx, c, ipfspinner.Any) +} + +// IsPinnedWithType returns whether or not the given cid is pinned with the +// given pin type, as well as returning the type of pin its pinned with. +func (p *pinner) IsPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (string, bool, error) { + p.lock.RLock() + defer p.lock.RUnlock() + return p.isPinnedWithType(ctx, c, mode) +} + +// isPinnedWithType is the implementation of IsPinnedWithType that does not lock. +// intended for use by other pinned methods that already take locks +func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (string, bool, error) { + switch mode { + case ipfspinner.Any, ipfspinner.Direct, ipfspinner.Indirect, ipfspinner.Recursive, ipfspinner.Internal: + default: + err := fmt.Errorf("invalid Pin Mode '%d', must be one of {%d, %d, %d, %d, %d}", + mode, ipfspinner.Direct, ipfspinner.Indirect, ipfspinner.Recursive, ipfspinner.Internal, ipfspinner.Any) + return "", false, err + } + if (mode == ipfspinner.Recursive || mode == ipfspinner.Any) && p.recursePin.Has(c) { + return linkRecursive, true, nil + } + if mode == ipfspinner.Recursive { + return "", false, nil + } + + if (mode == ipfspinner.Direct || mode == ipfspinner.Any) && p.directPin.Has(c) { + return linkDirect, true, nil + } + if mode == ipfspinner.Direct { + return "", false, nil + } + + if (mode == ipfspinner.Internal || mode == ipfspinner.Any) && p.isInternalPin(c) { + return linkInternal, true, nil + } + if mode == ipfspinner.Internal { + return "", false, nil + } + + // Default is Indirect + visitedSet := cid.NewSet() + for _, rc := range p.recursePin.Keys() { + has, err := hasChild(ctx, p.dserv, rc, c, visitedSet.Visit) + if err != nil { + return "", false, err + } + if has { + return rc.String(), true, nil + } + } + return "", false, nil +} + +// CheckIfPinned Checks if a set of keys are pinned, more efficient than +// calling IsPinned for each key, returns the pinned status of cid(s) +func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinner.Pinned, error) { + p.lock.RLock() + defer p.lock.RUnlock() + pinned := make([]ipfspinner.Pinned, 0, len(cids)) + toCheck := cid.NewSet() + + // First check for non-Indirect pins directly + for _, c := range cids { + if p.recursePin.Has(c) { + pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Recursive}) + } else if p.directPin.Has(c) { + pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Direct}) + } else if p.isInternalPin(c) { + pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Internal}) + } else { + toCheck.Add(c) + } + } + + // Now walk all recursive pins to check for indirect pins + var checkChildren func(cid.Cid, cid.Cid) error + checkChildren = func(rk, parentKey cid.Cid) error { + links, err := ipld.GetLinks(ctx, p.dserv, parentKey) + if err != nil { + return err + } + for _, lnk := range links { + c := lnk.Cid + + if toCheck.Has(c) { + pinned = append(pinned, + ipfspinner.Pinned{Key: c, Mode: ipfspinner.Indirect, Via: rk}) + toCheck.Remove(c) + } + + err := checkChildren(rk, c) + if err != nil { + return err + } + + if toCheck.Len() == 0 { + return nil + } + } + return nil + } + + for _, rk := range p.recursePin.Keys() { + err := checkChildren(rk, rk) + if err != nil { + return nil, err + } + if toCheck.Len() == 0 { + break + } + } + + // Anything left in toCheck is not pinned + for _, k := range toCheck.Keys() { + pinned = append(pinned, ipfspinner.Pinned{Key: k, Mode: ipfspinner.NotPinned}) + } + + return pinned, nil +} + +// RemovePinWithMode is for manually editing the pin structure. +// Use with care! If used improperly, garbage collection may not +// be successful. +func (p *pinner) RemovePinWithMode(c cid.Cid, mode ipfspinner.Mode) { + p.lock.Lock() + defer p.lock.Unlock() + switch mode { + case ipfspinner.Direct: + p.directPin.Remove(c) + case ipfspinner.Recursive: + p.recursePin.Remove(c) + default: + // programmer error, panic OK + panic("unrecognized pin type") + } +} + +func cidSetWithValues(cids []cid.Cid) *cid.Set { + out := cid.NewSet() + for _, c := range cids { + out.Add(c) + } + return out +} + +// DirectKeys returns a slice containing the directly pinned keys +func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.directPin.Keys(), nil +} + +// RecursiveKeys returns a slice containing the recursively pinned keys +func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.recursePin.Keys(), nil +} + +// Update updates a recursive pin from one cid to another +// this is more efficient than simply pinning the new one and unpinning the +// old one +func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error { + if from == to { + // Nothing to do. Don't remove this check or we'll end up + // _removing_ the pin. + // + // See #6648 + return nil + } + + p.lock.Lock() + defer p.lock.Unlock() + + if !p.recursePin.Has(from) { + return fmt.Errorf("'from' cid was not recursively pinned already") + } + + // Temporarily unlock while we fetch the differences. + p.lock.Unlock() + err := dagutils.DiffEnumerate(ctx, p.dserv, from, to) + p.lock.Lock() + + if err != nil { + return err + } + + p.recursePin.Add(to) + if unpin { + p.recursePin.Remove(from) + } + return nil +} + +// Flush encodes and writes pinner keysets to the datastore +func (p *pinner) Flush(ctx context.Context) error { + p.lock.Lock() + defer p.lock.Unlock() + + internalset := cid.NewSet() + recordInternal := internalset.Add + + root := &mdag.ProtoNode{} + { + n, err := storeSet(ctx, p.internal, p.directPin.Keys(), recordInternal) + if err != nil { + return err + } + if err := root.AddNodeLink(linkDirect, n); err != nil { + return err + } + } + + { + n, err := storeSet(ctx, p.internal, p.recursePin.Keys(), recordInternal) + if err != nil { + return err + } + if err := root.AddNodeLink(linkRecursive, n); err != nil { + return err + } + } + + // add the empty node, its referenced by the pin sets but never created + err := p.internal.Add(ctx, new(mdag.ProtoNode)) + if err != nil { + return err + } + + err = p.internal.Add(ctx, root) + if err != nil { + return err + } + + k := root.Cid() + + internalset.Add(k) + + if syncDServ, ok := p.dserv.(syncDAGService); ok { + if err := syncDServ.Sync(); err != nil { + return fmt.Errorf("cannot sync pinned data: %v", err) + } + } + + if syncInternal, ok := p.internal.(syncDAGService); ok { + if err := syncInternal.Sync(); err != nil { + return fmt.Errorf("cannot sync pinning data: %v", err) + } + } + + if err := p.dstore.Put(pinDatastoreKey, k.Bytes()); err != nil { + return fmt.Errorf("cannot store pin state: %v", err) + } + if err := p.dstore.Sync(pinDatastoreKey); err != nil { + return fmt.Errorf("cannot sync pin state: %v", err) + } + p.internalPin = internalset + return nil +} + +// InternalPins returns all cids kept pinned for the internal state of the +// pinner +func (p *pinner) InternalPins(ctx context.Context) ([]cid.Cid, error) { + p.lock.Lock() + defer p.lock.Unlock() + return p.internalPin.Keys(), nil +} + +// PinWithMode allows the user to have fine grained control over pin +// counts +func (p *pinner) PinWithMode(c cid.Cid, mode ipfspinner.Mode) { + p.lock.Lock() + defer p.lock.Unlock() + switch mode { + case ipfspinner.Recursive: + p.recursePin.Add(c) + case ipfspinner.Direct: + p.directPin.Add(c) + } +} + +// hasChild recursively looks for a Cid among the children of a root Cid. +// The visit function can be used to shortcut already-visited branches. +func hasChild(ctx context.Context, ng ipld.NodeGetter, root cid.Cid, child cid.Cid, visit func(cid.Cid) bool) (bool, error) { + links, err := ipld.GetLinks(ctx, ng, root) + if err != nil { + return false, err + } + for _, lnk := range links { + c := lnk.Cid + if lnk.Cid.Equals(child) { + return true, nil + } + if visit(c) { + has, err := hasChild(ctx, ng, c, child, visit) + if err != nil { + return false, err + } + + if has { + return has, nil + } + } + } + return false, nil +} diff --git a/pin_test.go b/ipldpinner/pin_test.go similarity index 90% rename from pin_test.go rename to ipldpinner/pin_test.go index e477ac0..e193aa9 100644 --- a/pin_test.go +++ b/ipldpinner/pin_test.go @@ -1,4 +1,4 @@ -package pin +package ipldpinner import ( "context" @@ -14,6 +14,7 @@ import ( dssync "github.com/ipfs/go-datastore/sync" blockstore "github.com/ipfs/go-ipfs-blockstore" offline "github.com/ipfs/go-ipfs-exchange-offline" + pin "github.com/ipfs/go-ipfs-pinner" util "github.com/ipfs/go-ipfs-util" ) @@ -30,7 +31,7 @@ func randNode() (*mdag.ProtoNode, cid.Cid) { return nd, k } -func assertPinned(t *testing.T, p Pinner, c cid.Cid, failmsg string) { +func assertPinned(t *testing.T, p pin.Pinner, c cid.Cid, failmsg string) { _, pinned, err := p.IsPinned(context.Background(), c) if err != nil { t.Fatal(err) @@ -41,7 +42,7 @@ func assertPinned(t *testing.T, p Pinner, c cid.Cid, failmsg string) { } } -func assertUnpinned(t *testing.T, p Pinner, c cid.Cid, failmsg string) { +func assertUnpinned(t *testing.T, p pin.Pinner, c cid.Cid, failmsg string) { _, pinned, err := p.IsPinned(context.Background(), c) if err != nil { t.Fatal(err) @@ -62,10 +63,13 @@ func TestPinnerBasic(t *testing.T) { dserv := mdag.NewDAGService(bserv) // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv, dserv) + p, err := New(dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } a, ak := randNode() - err := dserv.Add(ctx, a) + err = dserv.Add(ctx, a) if err != nil { t.Fatal(err) } @@ -151,7 +155,7 @@ func TestPinnerBasic(t *testing.T) { t.Fatal(err) } - np, err := LoadPinner(dstore, dserv, dserv) + np, err := New(dstore, dserv, dserv) if err != nil { t.Fatal(err) } @@ -188,7 +192,10 @@ func TestIsPinnedLookup(t *testing.T) { dserv := mdag.NewDAGService(bserv) // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv, dserv) + p, err := New(dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } aNodes := make([]*mdag.ProtoNode, aBranchLen) aKeys := make([]cid.Cid, aBranchLen) @@ -229,7 +236,7 @@ func TestIsPinnedLookup(t *testing.T) { } // Add C - err := dserv.Add(ctx, c) + err = dserv.Add(ctx, c) if err != nil { t.Fatal(err) } @@ -289,11 +296,13 @@ func TestDuplicateSemantics(t *testing.T) { dserv := mdag.NewDAGService(bserv) - // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv, dserv) + p, err := New(dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } a, _ := randNode() - err := dserv.Add(ctx, a) + err = dserv.Add(ctx, a) if err != nil { t.Fatal(err) } @@ -323,10 +332,13 @@ func TestFlush(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv, dserv) + p, err := New(dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } _, k := randNode() - p.PinWithMode(k, Recursive) + p.PinWithMode(k, pin.Recursive) if err := p.Flush(context.Background()); err != nil { t.Fatal(err) } @@ -340,11 +352,14 @@ func TestPinRecursiveFail(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv, dserv) + p, err := New(dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } a, _ := randNode() b, _ := randNode() - err := a.AddNodeLink("child", b) + err = a.AddNodeLink("child", b) if err != nil { t.Fatal(err) } @@ -385,7 +400,10 @@ func TestPinUpdate(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv, dserv) + p, err := New(dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } n1, c1 := randNode() n2, c2 := randNode() diff --git a/set.go b/ipldpinner/set.go similarity index 94% rename from set.go rename to ipldpinner/set.go index ca43797..2fb931f 100644 --- a/set.go +++ b/ipldpinner/set.go @@ -1,4 +1,4 @@ -package pin +package ipldpinner import ( "bytes" @@ -55,9 +55,14 @@ func (s sortByHash) Swap(a, b int) { } func storeItems(ctx context.Context, dag ipld.DAGService, estimatedLen uint64, depth uint32, iter itemIterator, internalKeys keyObserver) (*merkledag.ProtoNode, error) { - links := make([]*ipld.Link, 0, defaultFanout+maxItems) + // Each node wastes up to defaultFanout in empty links. + var leafLinks uint64 + if estimatedLen < maxItems { + leafLinks = estimatedLen + } + links := make([]*ipld.Link, defaultFanout, defaultFanout+leafLinks) for i := 0; i < defaultFanout; i++ { - links = append(links, &ipld.Link{Cid: emptyKey}) + links[i] = &ipld.Link{Cid: emptyKey} } // add emptyKey to our set of internal pinset objects @@ -97,7 +102,7 @@ func storeItems(ctx context.Context, dag ipld.DAGService, estimatedLen uint64, d sort.Stable(s) } - hashed := make([][]cid.Cid, defaultFanout) + var hashed [][]cid.Cid for { // This loop essentially enumerates every single item in the set // and maps them all into a set of buckets. Each bucket will be recursively @@ -116,6 +121,9 @@ func storeItems(ctx context.Context, dag ipld.DAGService, estimatedLen uint64, d if !ok { break } + if hashed == nil { + hashed = make([][]cid.Cid, defaultFanout) + } h := hash(depth, k) % defaultFanout hashed[h] = append(hashed[h], k) } diff --git a/set_test.go b/ipldpinner/set_test.go similarity index 99% rename from set_test.go rename to ipldpinner/set_test.go index 61a3118..0f32e6b 100644 --- a/set_test.go +++ b/ipldpinner/set_test.go @@ -1,4 +1,4 @@ -package pin +package ipldpinner import ( "context" diff --git a/pin.go b/pin.go index aa74c51..7e1d886 100644 --- a/pin.go +++ b/pin.go @@ -5,33 +5,14 @@ package pin import ( "context" "fmt" - "os" - "sync" - "time" cid "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" - mdag "github.com/ipfs/go-merkledag" - "github.com/ipfs/go-merkledag/dagutils" ) var log = logging.Logger("pin") -var pinDatastoreKey = ds.NewKey("/local/pins") - -var emptyKey cid.Cid - -func init() { - e, err := cid.Decode("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n") - if err != nil { - log.Error("failed to decode empty key constant") - os.Exit(1) - } - emptyKey = e -} - const ( linkRecursive = "recursive" linkDirect = "direct" @@ -177,482 +158,3 @@ func (p Pinned) String() string { return fmt.Sprintf("pinned: %s", modeStr) } } - -// pinner implements the Pinner interface -type pinner struct { - lock sync.RWMutex - recursePin *cid.Set - directPin *cid.Set - - // Track the keys used for storing the pinning state, so gc does - // not delete them. - internalPin *cid.Set - dserv ipld.DAGService - internal ipld.DAGService // dagservice used to store internal objects - dstore ds.Datastore -} - -type syncDAGService interface { - ipld.DAGService - Sync() error -} - -// NewPinner creates a new pinner using the given datastore as a backend -func NewPinner(dstore ds.Datastore, serv, internal ipld.DAGService) Pinner { - - rcset := cid.NewSet() - dirset := cid.NewSet() - - return &pinner{ - recursePin: rcset, - directPin: dirset, - dserv: serv, - dstore: dstore, - internal: internal, - internalPin: cid.NewSet(), - } -} - -// Pin the given node, optionally recursive -func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { - err := p.dserv.Add(ctx, node) - if err != nil { - return err - } - - c := node.Cid() - - p.lock.Lock() - defer p.lock.Unlock() - - if recurse { - if p.recursePin.Has(c) { - return nil - } - - p.lock.Unlock() - // temporary unlock to fetch the entire graph - err := mdag.FetchGraph(ctx, c, p.dserv) - p.lock.Lock() - if err != nil { - return err - } - - if p.recursePin.Has(c) { - return nil - } - - if p.directPin.Has(c) { - p.directPin.Remove(c) - } - - p.recursePin.Add(c) - } else { - if p.recursePin.Has(c) { - return fmt.Errorf("%s already pinned recursively", c.String()) - } - - p.directPin.Add(c) - } - return nil -} - -// ErrNotPinned is returned when trying to unpin items which are not pinned. -var ErrNotPinned = fmt.Errorf("not pinned or pinned indirectly") - -// Unpin a given key -func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { - p.lock.Lock() - defer p.lock.Unlock() - if p.recursePin.Has(c) { - if !recursive { - return fmt.Errorf("%s is pinned recursively", c) - } - p.recursePin.Remove(c) - return nil - } - if p.directPin.Has(c) { - p.directPin.Remove(c) - return nil - } - return ErrNotPinned -} - -func (p *pinner) isInternalPin(c cid.Cid) bool { - return p.internalPin.Has(c) -} - -// IsPinned returns whether or not the given key is pinned -// and an explanation of why its pinned -func (p *pinner) IsPinned(ctx context.Context, c cid.Cid) (string, bool, error) { - p.lock.RLock() - defer p.lock.RUnlock() - return p.isPinnedWithType(ctx, c, Any) -} - -// IsPinnedWithType returns whether or not the given cid is pinned with the -// given pin type, as well as returning the type of pin its pinned with. -func (p *pinner) IsPinnedWithType(ctx context.Context, c cid.Cid, mode Mode) (string, bool, error) { - p.lock.RLock() - defer p.lock.RUnlock() - return p.isPinnedWithType(ctx, c, mode) -} - -// isPinnedWithType is the implementation of IsPinnedWithType that does not lock. -// intended for use by other pinned methods that already take locks -func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode Mode) (string, bool, error) { - switch mode { - case Any, Direct, Indirect, Recursive, Internal: - default: - err := fmt.Errorf("invalid Pin Mode '%d', must be one of {%d, %d, %d, %d, %d}", - mode, Direct, Indirect, Recursive, Internal, Any) - return "", false, err - } - if (mode == Recursive || mode == Any) && p.recursePin.Has(c) { - return linkRecursive, true, nil - } - if mode == Recursive { - return "", false, nil - } - - if (mode == Direct || mode == Any) && p.directPin.Has(c) { - return linkDirect, true, nil - } - if mode == Direct { - return "", false, nil - } - - if (mode == Internal || mode == Any) && p.isInternalPin(c) { - return linkInternal, true, nil - } - if mode == Internal { - return "", false, nil - } - - // Default is Indirect - visitedSet := cid.NewSet() - for _, rc := range p.recursePin.Keys() { - has, err := hasChild(ctx, p.dserv, rc, c, visitedSet.Visit) - if err != nil { - return "", false, err - } - if has { - return rc.String(), true, nil - } - } - return "", false, nil -} - -// CheckIfPinned Checks if a set of keys are pinned, more efficient than -// calling IsPinned for each key, returns the pinned status of cid(s) -func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]Pinned, error) { - p.lock.RLock() - defer p.lock.RUnlock() - pinned := make([]Pinned, 0, len(cids)) - toCheck := cid.NewSet() - - // First check for non-Indirect pins directly - for _, c := range cids { - if p.recursePin.Has(c) { - pinned = append(pinned, Pinned{Key: c, Mode: Recursive}) - } else if p.directPin.Has(c) { - pinned = append(pinned, Pinned{Key: c, Mode: Direct}) - } else if p.isInternalPin(c) { - pinned = append(pinned, Pinned{Key: c, Mode: Internal}) - } else { - toCheck.Add(c) - } - } - - // Now walk all recursive pins to check for indirect pins - var checkChildren func(cid.Cid, cid.Cid) error - checkChildren = func(rk, parentKey cid.Cid) error { - links, err := ipld.GetLinks(ctx, p.dserv, parentKey) - if err != nil { - return err - } - for _, lnk := range links { - c := lnk.Cid - - if toCheck.Has(c) { - pinned = append(pinned, - Pinned{Key: c, Mode: Indirect, Via: rk}) - toCheck.Remove(c) - } - - err := checkChildren(rk, c) - if err != nil { - return err - } - - if toCheck.Len() == 0 { - return nil - } - } - return nil - } - - for _, rk := range p.recursePin.Keys() { - err := checkChildren(rk, rk) - if err != nil { - return nil, err - } - if toCheck.Len() == 0 { - break - } - } - - // Anything left in toCheck is not pinned - for _, k := range toCheck.Keys() { - pinned = append(pinned, Pinned{Key: k, Mode: NotPinned}) - } - - return pinned, nil -} - -// RemovePinWithMode is for manually editing the pin structure. -// Use with care! If used improperly, garbage collection may not -// be successful. -func (p *pinner) RemovePinWithMode(c cid.Cid, mode Mode) { - p.lock.Lock() - defer p.lock.Unlock() - switch mode { - case Direct: - p.directPin.Remove(c) - case Recursive: - p.recursePin.Remove(c) - default: - // programmer error, panic OK - panic("unrecognized pin type") - } -} - -func cidSetWithValues(cids []cid.Cid) *cid.Set { - out := cid.NewSet() - for _, c := range cids { - out.Add(c) - } - return out -} - -// LoadPinner loads a pinner and its keysets from the given datastore -func LoadPinner(d ds.Datastore, dserv, internal ipld.DAGService) (Pinner, error) { - p := new(pinner) - - rootKey, err := d.Get(pinDatastoreKey) - if err != nil { - return nil, fmt.Errorf("cannot load pin state: %v", err) - } - rootCid, err := cid.Cast(rootKey) - if err != nil { - return nil, err - } - - ctx, cancel := context.WithTimeout(context.TODO(), time.Second*5) - defer cancel() - - root, err := internal.Get(ctx, rootCid) - if err != nil { - return nil, fmt.Errorf("cannot find pinning root object: %v", err) - } - - rootpb, ok := root.(*mdag.ProtoNode) - if !ok { - return nil, mdag.ErrNotProtobuf - } - - internalset := cid.NewSet() - internalset.Add(rootCid) - recordInternal := internalset.Add - - { // load recursive set - recurseKeys, err := loadSet(ctx, internal, rootpb, linkRecursive, recordInternal) - if err != nil { - return nil, fmt.Errorf("cannot load recursive pins: %v", err) - } - p.recursePin = cidSetWithValues(recurseKeys) - } - - { // load direct set - directKeys, err := loadSet(ctx, internal, rootpb, linkDirect, recordInternal) - if err != nil { - return nil, fmt.Errorf("cannot load direct pins: %v", err) - } - p.directPin = cidSetWithValues(directKeys) - } - - p.internalPin = internalset - - // assign services - p.dserv = dserv - p.dstore = d - p.internal = internal - - return p, nil -} - -// DirectKeys returns a slice containing the directly pinned keys -func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { - p.lock.RLock() - defer p.lock.RUnlock() - - return p.directPin.Keys(), nil -} - -// RecursiveKeys returns a slice containing the recursively pinned keys -func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { - p.lock.RLock() - defer p.lock.RUnlock() - - return p.recursePin.Keys(), nil -} - -// Update updates a recursive pin from one cid to another -// this is more efficient than simply pinning the new one and unpinning the -// old one -func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error { - if from == to { - // Nothing to do. Don't remove this check or we'll end up - // _removing_ the pin. - // - // See #6648 - return nil - } - - p.lock.Lock() - defer p.lock.Unlock() - - if !p.recursePin.Has(from) { - return fmt.Errorf("'from' cid was not recursively pinned already") - } - - // Temporarily unlock while we fetch the differences. - p.lock.Unlock() - err := dagutils.DiffEnumerate(ctx, p.dserv, from, to) - p.lock.Lock() - - if err != nil { - return err - } - - p.recursePin.Add(to) - if unpin { - p.recursePin.Remove(from) - } - return nil -} - -// Flush encodes and writes pinner keysets to the datastore -func (p *pinner) Flush(ctx context.Context) error { - p.lock.Lock() - defer p.lock.Unlock() - - internalset := cid.NewSet() - recordInternal := internalset.Add - - root := &mdag.ProtoNode{} - { - n, err := storeSet(ctx, p.internal, p.directPin.Keys(), recordInternal) - if err != nil { - return err - } - if err := root.AddNodeLink(linkDirect, n); err != nil { - return err - } - } - - { - n, err := storeSet(ctx, p.internal, p.recursePin.Keys(), recordInternal) - if err != nil { - return err - } - if err := root.AddNodeLink(linkRecursive, n); err != nil { - return err - } - } - - // add the empty node, its referenced by the pin sets but never created - err := p.internal.Add(ctx, new(mdag.ProtoNode)) - if err != nil { - return err - } - - err = p.internal.Add(ctx, root) - if err != nil { - return err - } - - k := root.Cid() - - internalset.Add(k) - - if syncDServ, ok := p.dserv.(syncDAGService); ok { - if err := syncDServ.Sync(); err != nil { - return fmt.Errorf("cannot sync pinned data: %v", err) - } - } - - if syncInternal, ok := p.internal.(syncDAGService); ok { - if err := syncInternal.Sync(); err != nil { - return fmt.Errorf("cannot sync pinning data: %v", err) - } - } - - if err := p.dstore.Put(pinDatastoreKey, k.Bytes()); err != nil { - return fmt.Errorf("cannot store pin state: %v", err) - } - if err := p.dstore.Sync(pinDatastoreKey); err != nil { - return fmt.Errorf("cannot sync pin state: %v", err) - } - p.internalPin = internalset - return nil -} - -// InternalPins returns all cids kept pinned for the internal state of the -// pinner -func (p *pinner) InternalPins(ctx context.Context) ([]cid.Cid, error) { - p.lock.Lock() - defer p.lock.Unlock() - var out []cid.Cid - out = append(out, p.internalPin.Keys()...) - return out, nil -} - -// PinWithMode allows the user to have fine grained control over pin -// counts -func (p *pinner) PinWithMode(c cid.Cid, mode Mode) { - p.lock.Lock() - defer p.lock.Unlock() - switch mode { - case Recursive: - p.recursePin.Add(c) - case Direct: - p.directPin.Add(c) - } -} - -// hasChild recursively looks for a Cid among the children of a root Cid. -// The visit function can be used to shortcut already-visited branches. -func hasChild(ctx context.Context, ng ipld.NodeGetter, root cid.Cid, child cid.Cid, visit func(cid.Cid) bool) (bool, error) { - links, err := ipld.GetLinks(ctx, ng, root) - if err != nil { - return false, err - } - for _, lnk := range links { - c := lnk.Cid - if lnk.Cid.Equals(child) { - return true, nil - } - if visit(c) { - has, err := hasChild(ctx, ng, c, child, visit) - if err != nil { - return false, err - } - - if has { - return has, nil - } - } - } - return false, nil -} diff --git a/pinconv/pinconv.go b/pinconv/pinconv.go new file mode 100644 index 0000000..9aee703 --- /dev/null +++ b/pinconv/pinconv.go @@ -0,0 +1,128 @@ +// Package pinconv converts pins between the dag-based ipldpinner and the +// datastore-based dspinner. Once conversion is complete, the pins from the +// source pinner are removed. +package pinconv + +import ( + "context" + "fmt" + + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + ipfspinner "github.com/ipfs/go-ipfs-pinner" + "github.com/ipfs/go-ipfs-pinner/dspinner" + "github.com/ipfs/go-ipfs-pinner/ipldpinner" + ipld "github.com/ipfs/go-ipld-format" +) + +// ConvertPinsFromIPLDToDS converts pins stored in mdag based storage to pins +// stores in the datastore. Returns a dspinner loaded with the converted pins, +// and a count of the recursive and direct pins converted. +// +// After pins are stored in datastore, the root pin key is deleted to unlink +// the pin data in the DAGService. +func ConvertPinsFromIPLDToDS(ctx context.Context, dstore ds.Datastore, dserv ipld.DAGService, internal ipld.DAGService) (ipfspinner.Pinner, int, error) { + const ipldPinPath = "/local/pins" + + ipldPinner, err := ipldpinner.New(dstore, dserv, internal) + if err != nil { + return nil, 0, err + } + + dsPinner, err := dspinner.New(ctx, dstore, dserv) + if err != nil { + return nil, 0, err + } + + seen := cid.NewSet() + cids, err := ipldPinner.RecursiveKeys(ctx) + if err != nil { + return nil, 0, err + } + for i := range cids { + seen.Add(cids[i]) + dsPinner.PinWithMode(cids[i], ipfspinner.Recursive) + } + convCount := len(cids) + + cids, err = ipldPinner.DirectKeys(ctx) + if err != nil { + return nil, 0, err + } + for i := range cids { + if seen.Has(cids[i]) { + // Pin was already pinned recursively + continue + } + dsPinner.PinWithMode(cids[i], ipfspinner.Direct) + } + convCount += len(cids) + + err = dsPinner.Flush(ctx) + if err != nil { + return nil, 0, err + } + + // Delete root mdag key from datastore to remove old pin storage. + ipldPinDatastoreKey := ds.NewKey(ipldPinPath) + if err = dstore.Delete(ipldPinDatastoreKey); err != nil { + return nil, 0, fmt.Errorf("cannot delete old pin state: %v", err) + } + if err = dstore.Sync(ipldPinDatastoreKey); err != nil { + return nil, 0, fmt.Errorf("cannot sync old pin state: %v", err) + } + + return dsPinner, convCount, nil +} + +// ConvertPinsFromDSToIPLD converts the pins stored in the datastore by +// dspinner, into pins stored in the given internal DAGService by ipldpinner. +// Returns an ipldpinner loaded with the converted pins, and a count of the +// recursive and direct pins converted. +// +// After the pins are stored in the DAGService, the pins and their indexes are +// removed from the dspinner. +func ConvertPinsFromDSToIPLD(ctx context.Context, dstore ds.Datastore, dserv ipld.DAGService, internal ipld.DAGService) (ipfspinner.Pinner, int, error) { + dsPinner, err := dspinner.New(ctx, dstore, dserv) + if err != nil { + return nil, 0, err + } + + ipldPinner, err := ipldpinner.New(dstore, dserv, internal) + if err != nil { + return nil, 0, err + } + + cids, err := dsPinner.RecursiveKeys(ctx) + if err != nil { + return nil, 0, err + } + for i := range cids { + ipldPinner.PinWithMode(cids[i], ipfspinner.Recursive) + dsPinner.RemovePinWithMode(cids[i], ipfspinner.Recursive) + } + convCount := len(cids) + + cids, err = dsPinner.DirectKeys(ctx) + if err != nil { + return nil, 0, err + } + for i := range cids { + ipldPinner.PinWithMode(cids[i], ipfspinner.Direct) + dsPinner.RemovePinWithMode(cids[i], ipfspinner.Direct) + } + convCount += len(cids) + + // Save the ipldpinner pins + err = ipldPinner.Flush(ctx) + if err != nil { + return nil, 0, err + } + + err = dsPinner.Flush(ctx) + if err != nil { + return nil, 0, err + } + + return ipldPinner, convCount, nil +} diff --git a/pinconv/pinconv_test.go b/pinconv/pinconv_test.go new file mode 100644 index 0000000..ac7f8ff --- /dev/null +++ b/pinconv/pinconv_test.go @@ -0,0 +1,153 @@ +package pinconv + +import ( + "context" + "errors" + "io" + "testing" + + bs "github.com/ipfs/go-blockservice" + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + lds "github.com/ipfs/go-ds-leveldb" + blockstore "github.com/ipfs/go-ipfs-blockstore" + offline "github.com/ipfs/go-ipfs-exchange-offline" + ipfspin "github.com/ipfs/go-ipfs-pinner" + "github.com/ipfs/go-ipfs-pinner/dspinner" + util "github.com/ipfs/go-ipfs-util" + ipld "github.com/ipfs/go-ipld-format" + mdag "github.com/ipfs/go-merkledag" +) + +var rand = util.NewTimeSeededRand() + +type batchWrap struct { + ds.Datastore +} + +func randNode() (*mdag.ProtoNode, cid.Cid) { + nd := new(mdag.ProtoNode) + nd.SetData(make([]byte, 32)) + _, err := io.ReadFull(rand, nd.Data()) + if err != nil { + panic(err) + } + k := nd.Cid() + return nd, k +} + +func (d *batchWrap) Batch() (ds.Batch, error) { + return ds.NewBasicBatch(d), nil +} + +func makeStore() (ds.Datastore, ipld.DAGService) { + ldstore, err := lds.NewDatastore("", nil) + if err != nil { + panic(err) + } + var dstore ds.Batching + dstore = &batchWrap{ldstore} + + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + dserv := mdag.NewDAGService(bserv) + return dstore, dserv +} + +func TestConversions(t *testing.T) { + ctx := context.Background() + dstore, dserv := makeStore() + + dsPinner, err := dspinner.New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + a, ak := randNode() + err = dsPinner.Pin(ctx, a, false) + if err != nil { + t.Fatal(err) + } + + // create new node c, to be indirectly pinned through b + c, ck := randNode() + dserv.Add(ctx, c) + + // Create new node b, to be parent to a and c + b, _ := randNode() + b.AddNodeLink("child", a) + b.AddNodeLink("otherchild", c) + bk := b.Cid() // CID changed after adding links + + // recursively pin B{A,C} + err = dsPinner.Pin(ctx, b, true) + if err != nil { + t.Fatal(err) + } + + err = dsPinner.Flush(ctx) + if err != nil { + t.Fatal(err) + } + + verifyPins := func(pinner ipfspin.Pinner) error { + pinned, err := pinner.CheckIfPinned(ctx, ak, bk, ck) + if err != nil { + return err + } + if len(pinned) != 3 { + return errors.New("incorrect number of results") + } + for _, pn := range pinned { + switch pn.Key { + case ak: + if pn.Mode != ipfspin.Direct { + return errors.New("A pinned with wrong mode") + } + case bk: + if pn.Mode != ipfspin.Recursive { + return errors.New("B pinned with wrong mode") + } + case ck: + if pn.Mode != ipfspin.Indirect { + return errors.New("C should be pinned indirectly") + } + if pn.Via != bk { + return errors.New("C should be pinned via B") + } + } + } + return nil + } + + err = verifyPins(dsPinner) + if err != nil { + t.Fatal(err) + } + + ipldPinner, toIPLDCount, err := ConvertPinsFromDSToIPLD(ctx, dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } + if toIPLDCount != 2 { + t.Fatal("expected 2 ds-to-ipld pins, got", toIPLDCount) + } + + err = verifyPins(ipldPinner) + if err != nil { + t.Fatal(err) + } + + toDSPinner, toDSCount, err := ConvertPinsFromIPLDToDS(ctx, dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } + if toDSCount != toIPLDCount { + t.Fatal("ds-to-ipld pins", toIPLDCount, "not equal to ipld-to-ds-pins", toDSCount) + } + + err = verifyPins(toDSPinner) + if err != nil { + t.Fatal(err) + } +}