From ecc1d94b3abda33bd43e9a53a5227da345b51714 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 28 Oct 2020 22:17:01 +0100 Subject: [PATCH 1/4] Reduce badger ValueTreshold to 128 It should significntly size of the LSM index, and thus increase the performance with bigger datastores Signed-off-by: Jakub Sztandera --- node/repo/fsrepo_ds.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/repo/fsrepo_ds.go b/node/repo/fsrepo_ds.go index aa91d2514d0..28d737210a3 100644 --- a/node/repo/fsrepo_ds.go +++ b/node/repo/fsrepo_ds.go @@ -32,7 +32,7 @@ func chainBadgerDs(path string, readonly bool) (datastore.Batching, error) { opts.ReadOnly = readonly opts.Options = dgbadger.DefaultOptions("").WithTruncate(true). - WithValueThreshold(1 << 10) + WithValueThreshold(128) return badger.NewDatastore(path, &opts) } From 0297be4b9a70f65d7645b60af55948a48d4dd888 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 28 Oct 2020 23:24:45 +0100 Subject: [PATCH 2/4] Add `lotus-shed datastore rewrite` command Signed-off-by: Jakub Sztandera --- cmd/lotus-shed/datastore.go | 65 +++++++++++++++++++++++++++++++++++++ node/repo/fsrepo_ds.go | 23 +++++++------ 2 files changed, 78 insertions(+), 10 deletions(-) diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go index c6bac6815bf..85484534648 100644 --- a/cmd/lotus-shed/datastore.go +++ b/cmd/lotus-shed/datastore.go @@ -1,17 +1,22 @@ package main import ( + "bufio" "encoding/json" "fmt" + "io" "os" "strings" "github.com/docker/go-units" "github.com/ipfs/go-datastore" dsq "github.com/ipfs/go-datastore/query" + badgerds "github.com/ipfs/go-ds-badger2" logging "github.com/ipfs/go-log" + "github.com/mitchellh/go-homedir" "github.com/polydawn/refmt/cbor" "github.com/urfave/cli/v2" + "go.uber.org/multierr" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/lib/backupds" @@ -25,6 +30,7 @@ var datastoreCmd = &cli.Command{ datastoreBackupCmd, datastoreListCmd, datastoreGetCmd, + datastoreRewriteCmd, }, } @@ -288,3 +294,62 @@ func printVal(enc string, val []byte) error { return nil } + +var datastoreRewriteCmd = &cli.Command{ + Name: "rewrite", + Description: "rewrites badger datastore to compact it and possibly change params", + ArgsUsage: "source destination", + Action: func(cctx *cli.Context) error { + if cctx.NArg() != 2 { + return xerrors.Errorf("expected 2 arguments, got %d", cctx.NArg()) + } + fromPath, err := homedir.Expand(cctx.Args().Get(0)) + if err != nil { + return xerrors.Errorf("cannot get fromPath: %w", err) + } + toPath, err := homedir.Expand(cctx.Args().Get(1)) + if err != nil { + return xerrors.Errorf("cannot get toPath: %w", err) + } + + opts := repo.ChainBadgerOptions() + opts.Options = opts.Options.WithSyncWrites(false) + to, err := badgerds.NewDatastore(toPath, &opts) + if err != nil { + return xerrors.Errorf("opennig 'to' datastore: %w", err) + } + + opts.Options = opts.Options.WithReadOnly(false) + from, err := badgerds.NewDatastore(fromPath, &opts) + if err != nil { + return xerrors.Errorf("opennig 'from' datastore: %w", err) + } + + pr, pw := io.Pipe() + errCh := make(chan error) + go func() { + bw := bufio.NewWriterSize(pw, 64<<20) + _, err := from.DB.Backup(bw, 0) + bw.Flush() + pw.CloseWithError(err) + errCh <- err + }() + go func() { + err := to.DB.Load(pr, 256) + errCh <- err + }() + + err = <-errCh + if err != nil { + select { + case nerr := <-errCh: + err = multierr.Append(err, nerr) + default: + } + return err + } + + err = <-errCh + return err + }, +} diff --git a/node/repo/fsrepo_ds.go b/node/repo/fsrepo_ds.go index 28d737210a3..e7746cb8edc 100644 --- a/node/repo/fsrepo_ds.go +++ b/node/repo/fsrepo_ds.go @@ -4,18 +4,27 @@ import ( "os" "path/filepath" - "github.com/ipfs/go-datastore" + dgbadger "github.com/dgraph-io/badger/v2" + ldbopts "github.com/syndtr/goleveldb/leveldb/opt" "golang.org/x/xerrors" - dgbadger "github.com/dgraph-io/badger/v2" + "github.com/ipfs/go-datastore" badger "github.com/ipfs/go-ds-badger2" levelds "github.com/ipfs/go-ds-leveldb" measure "github.com/ipfs/go-ds-measure" - ldbopts "github.com/syndtr/goleveldb/leveldb/opt" ) type dsCtor func(path string, readonly bool) (datastore.Batching, error) +func ChainBadgerOptions() badger.Options { + opts := badger.DefaultOptions + opts.GcInterval = 0 // disable GC for chain datastore + + opts.Options = dgbadger.DefaultOptions("").WithTruncate(true). + WithValueThreshold(128) + return opts +} + var fsDatastores = map[string]dsCtor{ "chain": chainBadgerDs, "metadata": levelDs, @@ -27,13 +36,8 @@ var fsDatastores = map[string]dsCtor{ } func chainBadgerDs(path string, readonly bool) (datastore.Batching, error) { - opts := badger.DefaultOptions - opts.GcInterval = 0 // disable GC for chain datastore + opts := ChainBadgerOptions() opts.ReadOnly = readonly - - opts.Options = dgbadger.DefaultOptions("").WithTruncate(true). - WithValueThreshold(128) - return badger.NewDatastore(path, &opts) } @@ -43,7 +47,6 @@ func badgerDs(path string, readonly bool) (datastore.Batching, error) { opts.Options = dgbadger.DefaultOptions("").WithTruncate(true). WithValueThreshold(1 << 10) - return badger.NewDatastore(path, &opts) } From a2c66dd102741201302db03761853d8dabb68720 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 28 Oct 2020 23:30:28 +0100 Subject: [PATCH 3/4] make linter happy Signed-off-by: Jakub Sztandera --- cmd/lotus-shed/datastore.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go index 85484534648..9730d857f3f 100644 --- a/cmd/lotus-shed/datastore.go +++ b/cmd/lotus-shed/datastore.go @@ -330,8 +330,8 @@ var datastoreRewriteCmd = &cli.Command{ go func() { bw := bufio.NewWriterSize(pw, 64<<20) _, err := from.DB.Backup(bw, 0) - bw.Flush() - pw.CloseWithError(err) + _ = bw.Flush() + _ = pw.CloseWithError(err) errCh <- err }() go func() { From ae905bd0563744a183718260734a8b59b248c7fa Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 29 Oct 2020 01:05:31 +0100 Subject: [PATCH 4/4] Fix closing Signed-off-by: Jakub Sztandera --- cmd/lotus-shed/datastore.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go index 9730d857f3f..83422e77b82 100644 --- a/cmd/lotus-shed/datastore.go +++ b/cmd/lotus-shed/datastore.go @@ -350,6 +350,9 @@ var datastoreRewriteCmd = &cli.Command{ } err = <-errCh - return err + if err != nil { + return err + } + return multierr.Append(from.Close(), to.Close()) }, }