From 0e58b0c5d26d10b8698d54446484ba45a2c5aa4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 13 Jan 2018 22:28:24 +0100 Subject: [PATCH] make repo gc call CollectGarbage on datastore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/corerepo/gc.go | 4 ++-- core/coreunix/add_test.go | 2 +- package.json | 8 ++++---- pin/gc/gc.go | 15 ++++++++++++++- repo/fsrepo/datastores.go | 24 ++++++++++++++++++++++-- repo/fsrepo/fsrepo.go | 2 +- 6 files changed, 44 insertions(+), 11 deletions(-) diff --git a/core/corerepo/gc.go b/core/corerepo/gc.go index 2ea8db3d5e4..2f6739d9422 100644 --- a/core/corerepo/gc.go +++ b/core/corerepo/gc.go @@ -86,7 +86,7 @@ func GarbageCollect(n *core.IpfsNode, ctx context.Context) error { if err != nil { return err } - rmed := gc.GC(ctx, n.Blockstore, n.DAG, n.Pinning, roots) + rmed := gc.GC(ctx, n.Blockstore, n.Repo.Datastore(), n.DAG, n.Pinning, roots) return CollectResult(ctx, rmed, nil) } @@ -154,7 +154,7 @@ func GarbageCollectAsync(n *core.IpfsNode, ctx context.Context) <-chan gc.Result return out } - return gc.GC(ctx, n.Blockstore, n.DAG, n.Pinning, roots) + return gc.GC(ctx, n.Blockstore, n.Repo.Datastore(), n.DAG, n.Pinning, roots) } func PeriodicGC(ctx context.Context, node *core.IpfsNode) error { diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index 5ee2ef58ca8..91d79add499 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -102,7 +102,7 @@ func TestAddGCLive(t *testing.T) { gcstarted := make(chan struct{}) go func() { defer close(gcstarted) - gcout = gc.GC(context.Background(), node.Blockstore, node.DAG, node.Pinning, nil) + gcout = gc.GC(context.Background(), node.Blockstore, node.Repo.Datastore(), node.DAG, node.Pinning, nil) }() // gc shouldnt start until we let the add finish its current file. diff --git a/package.json b/package.json index 307d5151575..ea09c091854 100644 --- a/package.json +++ b/package.json @@ -208,9 +208,9 @@ }, { "author": "whyrusleeping", - "hash": "QmU7tt6mHJ5Wocjy2omBxpDfN8g9pkRimzJae7EXdrs96k", + "hash": "QmRhjB5Mnha4k6VH6qRFNabAVkxpbqC7bVw2daMKLHPXXN", "name": "go-ds-measure", - "version": "1.2.3" + "version": "1.3.0" }, { "author": "whyrusleeping", @@ -470,9 +470,9 @@ }, { "author": "magik6k", - "hash": "Qmdin8YL17fL1BC5ej6o9b8es6MBoiQjKVdyxEwJh3HVmf", + "hash": "Qmb1MciZErHUnT1MsyAwUL8FGtZqmRPAXjxU6SktqFUs23", "name": "go-ds-badger", - "version": "1.3.0" + "version": "1.4.1" }, { "author": "whyrusleeping", diff --git a/pin/gc/gc.go b/pin/gc/gc.go index f3b98597bcf..df36d4c98ff 100644 --- a/pin/gc/gc.go +++ b/pin/gc/gc.go @@ -11,6 +11,7 @@ import ( node "gx/ipfs/QmNwUEK7QbwSqyKBu3mMtToo8SUc6wQJ7gdZq4gGGJqfnf/go-ipld-format" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + dstore "gx/ipfs/QmdHG8MAuARdGHxx4rPQASLcvhz24fzjSQq7AJRAQEorq5/go-datastore" cid "gx/ipfs/QmeSrf6pzut73u6zLQkRFQ3ygt3k6XFT2kjdYP8Tnkwwyg/go-cid" ) @@ -33,7 +34,7 @@ type Result struct { // The routine then iterates over every block in the blockstore and // deletes any block that is not found in the marked set. // -func GC(ctx context.Context, bs bstore.GCBlockstore, ls dag.LinkService, pn pin.Pinner, bestEffortRoots []*cid.Cid) <-chan Result { +func GC(ctx context.Context, bs bstore.GCBlockstore, ds dstore.Datastore, ls dag.LinkService, pn pin.Pinner, bestEffortRoots []*cid.Cid) <-chan Result { elock := log.EventBegin(ctx, "GC.lockWait") unlocker := bs.GCLock() @@ -104,6 +105,18 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, ls dag.LinkService, pn pin. if errors { output <- Result{Error: ErrCannotDeleteSomeBlocks} } + + defer log.EventBegin(ctx, "GC.datastore").Done() + gds, ok := ds.(dstore.GCDatastore) + if !ok { + return + } + + err = gds.CollectGarbage() + if err != nil { + output <- Result{Error: err} + return + } }() return output diff --git a/repo/fsrepo/datastores.go b/repo/fsrepo/datastores.go index d2ad79804a8..9a934740109 100644 --- a/repo/fsrepo/datastores.go +++ b/repo/fsrepo/datastores.go @@ -10,15 +10,16 @@ import ( repo "github.com/ipfs/go-ipfs/repo" - measure "gx/ipfs/QmU7tt6mHJ5Wocjy2omBxpDfN8g9pkRimzJae7EXdrs96k/go-ds-measure" + measure "gx/ipfs/QmRhjB5Mnha4k6VH6qRFNabAVkxpbqC7bVw2daMKLHPXXN/go-ds-measure" flatfs "gx/ipfs/Qmak5iFUErGmKhYUC5StZtCGFp5bdhvye8PdKgTMk9B9Fw/go-ds-flatfs" ds "gx/ipfs/QmdHG8MAuARdGHxx4rPQASLcvhz24fzjSQq7AJRAQEorq5/go-datastore" mount "gx/ipfs/QmdHG8MAuARdGHxx4rPQASLcvhz24fzjSQq7AJRAQEorq5/go-datastore/syncmount" + humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" levelds "gx/ipfs/QmYnCBXxoyoS38vtNQjjpRwZTiUnpuuKpapxMNaDfyQRLf/go-ds-leveldb" + badgerds "gx/ipfs/Qmb1MciZErHUnT1MsyAwUL8FGtZqmRPAXjxU6SktqFUs23/go-ds-badger" ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt" - badgerds "gx/ipfs/Qmdin8YL17fL1BC5ej6o9b8es6MBoiQjKVdyxEwJh3HVmf/go-ds-badger" ) // ConfigFromMap creates a new datastore config from a map @@ -342,6 +343,8 @@ func (c measureDatastoreConfig) Create(path string) (repo.Datastore, error) { type badgerdsDatastoreConfig struct { path string syncWrites bool + + vlogFileSize int64 } // BadgerdsDatastoreConfig returns a configuration stub for a badger datastore @@ -366,6 +369,22 @@ func BadgerdsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, er } } + vls, ok := params["vlogFileSize"] + if !ok { + // default to 1GiB + c.vlogFileSize = badgerds.DefaultOptions.ValueLogFileSize + } else { + if vlogSize, ok := vls.(string); ok { + s, err := humanize.ParseBytes(vlogSize) + if err != nil { + return nil, err + } + c.vlogFileSize = int64(s) + } else { + return nil, fmt.Errorf("'vlogFileSize' field was not a string") + } + } + return &c, nil } @@ -389,6 +408,7 @@ func (c *badgerdsDatastoreConfig) Create(path string) (repo.Datastore, error) { defopts := badgerds.DefaultOptions defopts.SyncWrites = c.syncWrites + defopts.ValueLogFileSize = c.vlogFileSize return badgerds.NewDatastore(p, &defopts) } diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go index 51823042a77..a38c36a0c7f 100644 --- a/repo/fsrepo/fsrepo.go +++ b/repo/fsrepo/fsrepo.go @@ -24,8 +24,8 @@ import ( "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/mitchellh/go-homedir" util "gx/ipfs/QmPsAfmDBnZN3kZGSuNwvCNDZiHneERSKmRcFyG3UkvcT3/go-ipfs-util" + measure "gx/ipfs/QmRhjB5Mnha4k6VH6qRFNabAVkxpbqC7bVw2daMKLHPXXN/go-ds-measure" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" - measure "gx/ipfs/QmU7tt6mHJ5Wocjy2omBxpDfN8g9pkRimzJae7EXdrs96k/go-ds-measure" ma "gx/ipfs/QmW8s4zTsUoX1Q6CeYxVKPyqSKbF7H1YDUyTostBtZ8DaG/go-multiaddr" )