From 8562a9bb8208b70fcdf151852d1e0dde5e1f99e8 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 8 Mar 2021 18:12:09 +0200 Subject: [PATCH 1/4] garbage collect hotstore after compaction --- blockstore/badger/blockstore.go | 9 +++++++++ blockstore/splitstore/splitstore.go | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/blockstore/badger/blockstore.go b/blockstore/badger/blockstore.go index cd740e650f6..7c249880c6a 100644 --- a/blockstore/badger/blockstore.go +++ b/blockstore/badger/blockstore.go @@ -131,6 +131,15 @@ func (b *Blockstore) Close() error { return b.DB.Close() } +// GC runs garbage collection on the value log +func (b *Blockstore) GC() error { + if atomic.LoadInt64(&b.state) != stateOpen { + return ErrBlockstoreClosed + } + + return b.DB.RunValueLogGC(0.125) +} + // View implements blockstore.Viewer, which leverages zero-copy read-only // access to values. func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error { diff --git a/blockstore/splitstore/splitstore.go b/blockstore/splitstore/splitstore.go index 5ed64b54ec9..c69e26cc29f 100644 --- a/blockstore/splitstore/splitstore.go +++ b/blockstore/splitstore/splitstore.go @@ -710,6 +710,17 @@ func (s *SplitStore) compactSimple(curTs *types.TipSet) error { return xerrors.Errorf("error syncing tracker: %w", err) } + if gc, ok := s.hot.(interface{ GC() error }); ok { + log.Infof("garbage collecting hotstore") + startGC := time.Now() + err = gc.GC() + if err != nil { + log.Warnf("error garbage collecting hotstore: %s", err) + } else { + log.Infow("garbage collecting done", "took", time.Since(startGC)) + } + } + err = s.setBaseEpoch(coldEpoch) if err != nil { return xerrors.Errorf("error saving base epoch: %w", err) From 52de95d3443841d0b51594544cf744317042c139 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 8 Mar 2021 18:30:09 +0200 Subject: [PATCH 2/4] also gc in compactFull, not just compactSimple --- blockstore/splitstore/splitstore.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/blockstore/splitstore/splitstore.go b/blockstore/splitstore/splitstore.go index c69e26cc29f..6551b6d51a4 100644 --- a/blockstore/splitstore/splitstore.go +++ b/blockstore/splitstore/splitstore.go @@ -1016,6 +1016,17 @@ func (s *SplitStore) compactFull(curTs *types.TipSet) error { return xerrors.Errorf("error syncing tracker: %w", err) } + if gc, ok := s.hot.(interface{ GC() error }); ok { + log.Infof("garbage collecting hotstore") + startGC := time.Now() + err = gc.GC() + if err != nil { + log.Warnf("error garbage collecting hotstore: %s", err) + } else { + log.Infow("garbage collecting done", "took", time.Since(startGC)) + } + } + err = s.setBaseEpoch(coldEpoch) if err != nil { return xerrors.Errorf("error saving base epoch: %w", err) From 3d1b855f20149c4e5ff6d7493d0a1c45e49706ef Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 8 Mar 2021 19:22:53 +0200 Subject: [PATCH 3/4] rename GC to CollectGarbage, ignore badger.ErrNoRewrite --- blockstore/badger/blockstore.go | 12 +++++++++--- blockstore/splitstore/splitstore.go | 12 ++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/blockstore/badger/blockstore.go b/blockstore/badger/blockstore.go index 7c249880c6a..16f9331f265 100644 --- a/blockstore/badger/blockstore.go +++ b/blockstore/badger/blockstore.go @@ -131,13 +131,19 @@ func (b *Blockstore) Close() error { return b.DB.Close() } -// GC runs garbage collection on the value log -func (b *Blockstore) GC() error { +// CollectGarbage runs garbage collection on the value log +func (b *Blockstore) CollectGarbage() error { if atomic.LoadInt64(&b.state) != stateOpen { return ErrBlockstoreClosed } - return b.DB.RunValueLogGC(0.125) + err := b.DB.RunValueLogGC(0.125) + if err == badger.ErrNoRewrite { + // not really an error in this case + return nil + } + + return err } // View implements blockstore.Viewer, which leverages zero-copy read-only diff --git a/blockstore/splitstore/splitstore.go b/blockstore/splitstore/splitstore.go index 6551b6d51a4..4e20dd7d298 100644 --- a/blockstore/splitstore/splitstore.go +++ b/blockstore/splitstore/splitstore.go @@ -710,14 +710,14 @@ func (s *SplitStore) compactSimple(curTs *types.TipSet) error { return xerrors.Errorf("error syncing tracker: %w", err) } - if gc, ok := s.hot.(interface{ GC() error }); ok { + if gc, ok := s.hot.(interface{ CollectGarbage() error }); ok { log.Infof("garbage collecting hotstore") startGC := time.Now() - err = gc.GC() + err = gc.CollectGarbage() if err != nil { log.Warnf("error garbage collecting hotstore: %s", err) } else { - log.Infow("garbage collecting done", "took", time.Since(startGC)) + log.Infow("garbage collection done", "took", time.Since(startGC)) } } @@ -1016,14 +1016,14 @@ func (s *SplitStore) compactFull(curTs *types.TipSet) error { return xerrors.Errorf("error syncing tracker: %w", err) } - if gc, ok := s.hot.(interface{ GC() error }); ok { + if gc, ok := s.hot.(interface{ CollectGarbage() error }); ok { log.Infof("garbage collecting hotstore") startGC := time.Now() - err = gc.GC() + err = gc.CollectGarbage() if err != nil { log.Warnf("error garbage collecting hotstore: %s", err) } else { - log.Infow("garbage collecting done", "took", time.Since(startGC)) + log.Infow("garbage collection done", "took", time.Since(startGC)) } } From 3bd77701d8acdb3f0eb304107452bcecbb71db50 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 8 Mar 2021 19:46:21 +0200 Subject: [PATCH 4/4] deduplicate code --- blockstore/splitstore/splitstore.go | 35 +++++++++++++---------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/blockstore/splitstore/splitstore.go b/blockstore/splitstore/splitstore.go index 4e20dd7d298..fb3e2880315 100644 --- a/blockstore/splitstore/splitstore.go +++ b/blockstore/splitstore/splitstore.go @@ -710,16 +710,7 @@ func (s *SplitStore) compactSimple(curTs *types.TipSet) error { return xerrors.Errorf("error syncing tracker: %w", err) } - if gc, ok := s.hot.(interface{ CollectGarbage() error }); ok { - log.Infof("garbage collecting hotstore") - startGC := time.Now() - err = gc.CollectGarbage() - if err != nil { - log.Warnf("error garbage collecting hotstore: %s", err) - } else { - log.Infow("garbage collection done", "took", time.Since(startGC)) - } - } + s.gcHotstore() err = s.setBaseEpoch(coldEpoch) if err != nil { @@ -806,6 +797,19 @@ func (s *SplitStore) purgeTracking(cids []cid.Cid) error { return s.purgeBatch(cids, s.tracker.DeleteBatch) } +func (s *SplitStore) gcHotstore() { + if gc, ok := s.hot.(interface{ CollectGarbage() error }); ok { + log.Infof("garbage collecting hotstore") + startGC := time.Now() + err := gc.CollectGarbage() + if err != nil { + log.Warnf("error garbage collecting hotstore: %s", err) + } else { + log.Infow("garbage collection done", "took", time.Since(startGC)) + } + } +} + func (s *SplitStore) compactFull(curTs *types.TipSet) error { currentEpoch := curTs.Height() coldEpoch := s.baseEpoch + CompactionCold @@ -1016,16 +1020,7 @@ func (s *SplitStore) compactFull(curTs *types.TipSet) error { return xerrors.Errorf("error syncing tracker: %w", err) } - if gc, ok := s.hot.(interface{ CollectGarbage() error }); ok { - log.Infof("garbage collecting hotstore") - startGC := time.Now() - err = gc.CollectGarbage() - if err != nil { - log.Warnf("error garbage collecting hotstore: %s", err) - } else { - log.Infow("garbage collection done", "took", time.Since(startGC)) - } - } + s.gcHotstore() err = s.setBaseEpoch(coldEpoch) if err != nil {