Skip to content

Commit

Permalink
fix: Separate chain and state stores.
Browse files Browse the repository at this point in the history
  • Loading branch information
strahe committed Jan 19, 2022
1 parent c0f29ea commit 201e834
Show file tree
Hide file tree
Showing 6 changed files with 83 additions and 21 deletions.
8 changes: 8 additions & 0 deletions blockstore/splitstore/splitstore.go
Original file line number Diff line number Diff line change
Expand Up @@ -611,3 +611,11 @@ func (s *SplitStore) setBaseEpoch(epoch abi.ChainEpoch) error {
s.baseEpoch = epoch
return s.ds.Put(s.ctx, baseEpochKey, epochToBytes(epoch))
}

func (s *SplitStore) HotBlockStore() bstore.Blockstore {
return s.hot.(bstore.Blockstore)
}

func (s *SplitStore) ColdBlockStore() bstore.Blockstore {
return s.cold.(bstore.Blockstore)
}
4 changes: 2 additions & 2 deletions chain/store/messages.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func (cs *ChainStore) GetSignedMessage(ctx context.Context, c cid.Cid) (*types.S
func (cs *ChainStore) readAMTCids(root cid.Cid) ([]cid.Cid, error) {
ctx := context.TODO()
// block headers use adt0, for now.
a, err := blockadt.AsArray(cs.ActorStore(ctx), root)
a, err := blockadt.AsArray(cs.ChainAdtStore(ctx), root)
if err != nil {
return nil, xerrors.Errorf("amt load: %w", err)
}
Expand Down Expand Up @@ -258,7 +258,7 @@ func (cs *ChainStore) MessagesForBlock(ctx context.Context, b *types.BlockHeader

func (cs *ChainStore) GetParentReceipt(ctx context.Context, b *types.BlockHeader, i int) (*types.MessageReceipt, error) {
// block headers use adt0, for now.
a, err := blockadt.AsArray(cs.ActorStore(ctx), b.ParentMessageReceipts)
a, err := blockadt.AsArray(cs.ChainAdtStore(ctx), b.ParentMessageReceipts)
if err != nil {
return nil, xerrors.Errorf("amt load: %w", err)
}
Expand Down
4 changes: 4 additions & 0 deletions chain/store/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -1108,6 +1108,10 @@ func (cs *ChainStore) ActorStore(ctx context.Context) adt.Store {
return ActorStore(ctx, cs.stateBlockstore)
}

func (cs *ChainStore) ChainAdtStore(ctx context.Context) adt.Store {
return adt.WrapStore(ctx, cbor.NewCborStore(cs.chainLocalBlockstore))
}

func (cs *ChainStore) TryFillTipSet(ctx context.Context, ts *types.TipSet) (*FullTipSet, error) {
var out []*types.FullBlock

Expand Down
2 changes: 2 additions & 0 deletions node/builder_chain.go
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,8 @@ func ConfigFullNode(c interface{}) Option {
Override(new(dtypes.ExposedBlockstore), From(new(dtypes.UniversalBlockstore))),
Override(new(dtypes.GCReferenceProtector), modules.NoopGCReferenceProtector),
),
If(os.Getenv("LOTUS_SEPARATE_CHAINSTORE_STATESTORE") == "1",
Override(new(dtypes.BasicStateBlockstore), modules.BadgerStateBlockstore)),

Override(new(dtypes.ChainBlockstore), From(new(dtypes.BasicChainBlockstore))),
Override(new(dtypes.StateBlockstore), From(new(dtypes.BasicStateBlockstore))),
Expand Down
23 changes: 23 additions & 0 deletions node/modules/blockstore.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,29 @@ func BadgerHotBlockstore(lc fx.Lifecycle, r repo.LockedRepo) (dtypes.HotBlocksto
return bs, nil
}

func BadgerStateBlockstore(lc fx.Lifecycle, r repo.LockedRepo) (dtypes.BasicStateBlockstore, error) {
path := filepath.Join(r.Path(), "datastore", "state")
if err := os.MkdirAll(path, 0755); err != nil {
return nil, err
}

opts := badgerbs.DefaultOptions(path)
opts.ReadOnly = r.Readonly()

bs, err := badgerbs.Open(opts)
if err != nil {
return nil, err
}

lc.Append(fx.Hook{
OnStop: func(_ context.Context) error {
return bs.Close()
}})

return bs, nil
}


func SplitBlockstore(cfg *config.Chainstore) func(lc fx.Lifecycle, r repo.LockedRepo, ds dtypes.MetadataDS, cold dtypes.ColdBlockstore, hot dtypes.HotBlockstore) (dtypes.SplitBlockstore, error) {
return func(lc fx.Lifecycle, r repo.LockedRepo, ds dtypes.MetadataDS, cold dtypes.ColdBlockstore, hot dtypes.HotBlockstore) (dtypes.SplitBlockstore, error) {
path, err := r.SplitstorePath()
Expand Down
63 changes: 44 additions & 19 deletions node/modules/genesis.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ package modules

import (
"bytes"
"context"
"github.com/filecoin-project/lotus/blockstore"
"golang.org/x/sync/errgroup"
"os"

"go.uber.org/fx"
Expand All @@ -22,31 +25,53 @@ func ErrorGenesis() Genesis {
}
}

func LoadGenesis(genBytes []byte) func(fx.Lifecycle, helpers.MetricsCtx, dtypes.ChainBlockstore) Genesis {
return func(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtypes.ChainBlockstore) Genesis {
func LoadGenesis(genBytes []byte) func(fx.Lifecycle, helpers.MetricsCtx, dtypes.ChainBlockstore, dtypes.StateBlockstore) Genesis {
return func(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtypes.ChainBlockstore, ss dtypes.StateBlockstore) Genesis {
return func() (header *types.BlockHeader, e error) {
ctx := helpers.LifecycleCtx(mctx, lc)
c, err := car.LoadCar(ctx, bs, bytes.NewReader(genBytes))
if err != nil {
return nil, xerrors.Errorf("loading genesis car file failed: %w", err)
}
if len(c.Roots) != 1 {
return nil, xerrors.New("expected genesis file to have one root")
}
root, err := bs.Get(ctx, c.Roots[0])
if err != nil {
return nil, err
}

h, err := types.DecodeBlock(root.RawData())
if err != nil {
return nil, xerrors.Errorf("decoding block failed: %w", err)
}
return h, nil
genBytes2 := make([]byte, len(genBytes))
copy(genBytes2, genBytes)
grp, _ := errgroup.WithContext(context.TODO())
grp.Go(func() error {
h, err := loadGenesis(ctx, bs, genBytes)
if err == nil {
header = h
}
return err
})
grp.Go(func() error {
h, err := loadGenesis(ctx, ss, genBytes2)
if err == nil {
header = h
}
return err
})
e = grp.Wait()
return
}
}
}

func loadGenesis(ctx context.Context, bs blockstore.Blockstore, genBytes []byte) (header *types.BlockHeader, e error) {
c, err := car.LoadCar(ctx, bs, bytes.NewReader(genBytes))
if err != nil {
return nil, xerrors.Errorf("loading genesis car file failed: %w", err)
}
if len(c.Roots) != 1 {
return nil, xerrors.New("expected genesis file to have one root")
}
root, err := bs.Get(ctx, c.Roots[0])
if err != nil {
return nil, err
}

h, err := types.DecodeBlock(root.RawData())
if err != nil {
return nil, xerrors.Errorf("decoding block failed: %w", err)
}
return h, nil
}

func DoSetGenesis(_ dtypes.AfterGenesisSet) {}

func SetGenesis(lc fx.Lifecycle, mctx helpers.MetricsCtx, cs *store.ChainStore, g Genesis) (dtypes.AfterGenesisSet, error) {
Expand Down

0 comments on commit 201e834

Please sign in to comment.