From 734db298635f7a48395bb7f592c56b7fd2fdefec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 21 Mar 2024 15:07:49 +0100 Subject: [PATCH] feat: shed: Add v13 migration to migrate-state (#11601) * shed: Add v13 migration to migrate-state * shed: some ADL tools, update GST * shed: diff hamt address mode * shed migration debug tooling * shed migration: market diff on error * shed: Fix cached migration diff * shed: Diff deal states on bad migration * shed: Use std json * shed: Drill in the migration diff more * shed: Show proposals in migration market diff * shed: Show added provider sectors diff * shed: hamts are hard to use * update gst * update gst * update gst * update GST with fixed invartiant * go mod tidy --- cmd/lotus-shed/adl.go | 124 +++++++++++ cmd/lotus-shed/diff.go | 261 ++++++++++++++++++++++- cmd/lotus-shed/main.go | 1 + cmd/lotus-shed/migrations.go | 396 +++++++++++++++++++++++++++++++++++ go.mod | 2 +- 5 files changed, 782 insertions(+), 2 deletions(-) create mode 100644 cmd/lotus-shed/adl.go diff --git a/cmd/lotus-shed/adl.go b/cmd/lotus-shed/adl.go new file mode 100644 index 00000000000..762f78b6c82 --- /dev/null +++ b/cmd/lotus-shed/adl.go @@ -0,0 +1,124 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + "github.com/ipld/go-car" + "github.com/urfave/cli/v2" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" + + adt13 "github.com/filecoin-project/go-state-types/builtin/v13/util/adt" + + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/actors/adt" +) + +var adlCmd = &cli.Command{ + Name: "adl", + Usage: "adl manipulation commands", + Subcommands: []*cli.Command{ + adlAmtCmd, + }, +} + +var adlAmtCmd = &cli.Command{ + Name: "amt", + Usage: "AMT manipulation commands", + Subcommands: []*cli.Command{ + adlAmtGetCmd, + }, +} + +var adlAmtGetCmd = &cli.Command{ + Name: "get", + Usage: "Get an element from an AMT", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "car-file", + Usage: "write a car file with two hamts (use lotus-shed export-car)", + }, + &cli.IntFlag{ + Name: "bitwidth", + Usage: "bitwidth of the HAMT", + Value: 5, + }, + &cli.StringFlag{ + Name: "root", + Usage: "root cid of the HAMT", + }, + &cli.Int64Flag{ + Name: "key", + Usage: "key to get", + }, + }, + Action: func(cctx *cli.Context) error { + bs := blockstore.NewMemorySync() + + f, err := os.Open(cctx.String("car-file")) + if err != nil { + return err + } + defer func(f *os.File) { + _ = f.Close() + }(f) + + cr, err := car.NewCarReader(f) + if err != nil { + return err + } + + for { + blk, err := cr.Next() + if err != nil { + if err == io.EOF { + break + } + return err + } + + if err := bs.Put(cctx.Context, blk); err != nil { + return err + } + } + + root, err := cid.Parse(cctx.String("root")) + if err != nil { + return err + } + + m, err := adt13.AsArray(adt.WrapStore(cctx.Context, cbor.NewCborStore(bs)), root, cctx.Int("bitwidth")) + if err != nil { + return err + } + + var out cbg.Deferred + ok, err := m.Get(cctx.Uint64("key"), &out) + if err != nil { + return err + } + if !ok { + return xerrors.Errorf("no such element") + } + + fmt.Printf("RAW: %x\n", out.Raw) + fmt.Println("----") + + var i interface{} + if err := cbor.DecodeInto(out.Raw, &i); err == nil { + ij, err := json.MarshalIndent(i, "", " ") + if err != nil { + return err + } + + fmt.Println(string(ij)) + } + + return nil + }, +} diff --git a/cmd/lotus-shed/diff.go b/cmd/lotus-shed/diff.go index 981dc850c62..a8eac657514 100644 --- a/cmd/lotus-shed/diff.go +++ b/cmd/lotus-shed/diff.go @@ -1,20 +1,31 @@ package main import ( + "bytes" "context" + "encoding/json" "fmt" "io" + "os" + "github.com/fatih/color" "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + "github.com/ipld/go-car" "github.com/urfave/cli/v2" "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-amt-ipld/v4" + "github.com/filecoin-project/go-hamt-ipld/v3" "github.com/filecoin-project/go-state-types/abi" miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" + "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/must" "github.com/filecoin-project/lotus/node/repo" ) @@ -24,6 +35,8 @@ var diffCmd = &cli.Command{ Subcommands: []*cli.Command{ diffStateTrees, diffMinerStates, + diffHAMTs, + diffAMTs, }, } @@ -64,7 +77,9 @@ var diffMinerStates = &cli.Command{ return err } - defer lkrepo.Close() //nolint:errcheck + defer func(lkrepo repo.LockedRepo) { + _ = lkrepo.Close() + }(lkrepo) bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { @@ -258,3 +273,247 @@ var diffStateTrees = &cli.Command{ return nil }, } + +var diffHAMTs = &cli.Command{ + Name: "hamts", + Usage: "diff two HAMTs", + ArgsUsage: " ", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "car-file", + Usage: "write a car file with two hamts (use lotus-shed export-car)", + }, + &cli.IntFlag{ + Name: "bitwidth", + Usage: "bitwidth of the HAMT", + Value: 5, + }, + &cli.StringFlag{ + Name: "key-type", + Usage: "type of the key", + Value: "uint", + }, + }, + Action: func(cctx *cli.Context) error { + var bs blockstore.Blockstore = blockstore.NewMemorySync() + + if cctx.IsSet("car-file") { + f, err := os.Open(cctx.String("car-file")) + if err != nil { + return err + } + defer func(f *os.File) { + _ = f.Close() + }(f) + + cr, err := car.NewCarReader(f) + if err != nil { + return err + } + + for { + blk, err := cr.Next() + if err != nil { + if err == io.EOF { + break + } + return err + } + + if err := bs.Put(cctx.Context, blk); err != nil { + return err + } + } + } else { + // use running node + api, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return xerrors.Errorf("connect to full node: %w", err) + } + defer closer() + + bs = blockstore.NewAPIBlockstore(api) + } + + cidA, err := cid.Parse(cctx.Args().Get(0)) + if err != nil { + return err + } + + cidB, err := cid.Parse(cctx.Args().Get(1)) + if err != nil { + return err + } + + cst := cbor.NewCborStore(bs) + + var keyParser func(k string) (interface{}, error) + switch cctx.String("key-type") { + case "uint": + keyParser = func(k string) (interface{}, error) { + return abi.ParseUIntKey(k) + } + case "actor": + keyParser = func(k string) (interface{}, error) { + return address.NewFromBytes([]byte(k)) + } + default: + return fmt.Errorf("unknown key type: %s", cctx.String("key-type")) + } + + diffs, err := hamt.Diff(cctx.Context, cst, cst, cidA, cidB, hamt.UseTreeBitWidth(cctx.Int("bitwidth"))) + if err != nil { + return err + } + + for _, d := range diffs { + switch d.Type { + case hamt.Add: + color.Green("+ Add %v", must.One(keyParser(d.Key))) + case hamt.Remove: + color.Red("- Remove %v", must.One(keyParser(d.Key))) + case hamt.Modify: + color.Yellow("~ Modify %v", must.One(keyParser(d.Key))) + } + } + + return nil + }, +} + +var diffAMTs = &cli.Command{ + Name: "amts", + Usage: "diff two AMTs", + ArgsUsage: " ", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "car-file", + Usage: "write a car file with two amts (use lotus-shed export-car)", + }, + &cli.UintFlag{ + Name: "bitwidth", + Usage: "bitwidth of the AMT", + Value: 5, + }, + }, + Action: func(cctx *cli.Context) error { + var bs blockstore.Blockstore = blockstore.NewMemorySync() + + if cctx.IsSet("car-file") { + f, err := os.Open(cctx.String("car-file")) + if err != nil { + return err + } + defer func(f *os.File) { + _ = f.Close() + }(f) + + cr, err := car.NewCarReader(f) + if err != nil { + return err + } + + for { + blk, err := cr.Next() + if err != nil { + if err == io.EOF { + break + } + return err + } + + if err := bs.Put(cctx.Context, blk); err != nil { + return err + } + } + } else { + // use running node + api, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return xerrors.Errorf("connect to full node: %w", err) + } + defer closer() + + bs = blockstore.NewAPIBlockstore(api) + } + + cidA, err := cid.Parse(cctx.Args().Get(0)) + if err != nil { + return err + } + + cidB, err := cid.Parse(cctx.Args().Get(1)) + if err != nil { + return err + } + + cst := cbor.NewCborStore(bs) + + diffs, err := amt.Diff(cctx.Context, cst, cst, cidA, cidB, amt.UseTreeBitWidth(cctx.Uint("bitwidth"))) + if err != nil { + return err + } + + for _, d := range diffs { + switch d.Type { + case amt.Add: + color.Green("+ Add %v", d.Key) + case amt.Remove: + color.Red("- Remove %v", d.Key) + case amt.Modify: + color.Yellow("~ Modify %v", d.Key) + + var vb, va interface{} + err := cbor.DecodeInto(d.Before.Raw, &vb) + if err != nil { + return err + } + err = cbor.DecodeInto(d.After.Raw, &va) + if err != nil { + return err + } + + vjsonb, err := json.MarshalIndent(vb, " ", " ") + if err != nil { + return err + } + vjsona, err := json.MarshalIndent(va, " ", " ") + if err != nil { + return err + } + + linesb := bytes.Split(vjsonb, []byte("\n")) // - + linesa := bytes.Split(vjsona, []byte("\n")) // + + + maxLen := len(linesb) + if len(linesa) > maxLen { + maxLen = len(linesa) + } + + for i := 0; i < maxLen; i++ { + // Check if 'linesb' has run out of lines but 'linesa' hasn't + if i >= len(linesb) && i < len(linesa) { + color.Green("+ %s\n", linesa[i]) + continue + } + // Check if 'linesa' has run out of lines but 'linesb' hasn't + if i >= len(linesa) && i < len(linesb) { + color.Red("- %s\n", linesb[i]) + continue + } + // Compare lines if both slices have lines at index i + if !bytes.Equal(linesb[i], linesa[i]) { + color.Red("- %s\n", linesb[i]) + color.Green("+ %s\n", linesa[i]) + } else { + // Print the line if it is the same in both slices + fmt.Printf(" %s\n", linesb[i]) + } + } + + } + } + + return nil + }, +} diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index e3b1333edbd..2b3b18670da 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -91,6 +91,7 @@ func main() { FevmAnalyticsCmd, mismatchesCmd, blockCmd, + adlCmd, } app := &cli.App{ diff --git a/cmd/lotus-shed/migrations.go b/cmd/lotus-shed/migrations.go index 96e4747b7ef..febe833d75e 100644 --- a/cmd/lotus-shed/migrations.go +++ b/cmd/lotus-shed/migrations.go @@ -1,27 +1,41 @@ package main import ( + "bytes" "context" + "encoding/json" "fmt" "os" "path/filepath" "strconv" "time" + "github.com/fatih/color" + "github.com/ipfs/boxo/blockservice" + "github.com/ipfs/boxo/exchange/offline" + "github.com/ipfs/boxo/ipld/merkledag" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" + cbornode "github.com/ipfs/go-ipld-cbor" + "github.com/ipld/go-car" "github.com/urfave/cli/v2" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" ffi "github.com/filecoin-project/filecoin-ffi" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-amt-ipld/v4" + "github.com/filecoin-project/go-hamt-ipld/v3" "github.com/filecoin-project/go-state-types/abi" actorstypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/builtin" v10 "github.com/filecoin-project/go-state-types/builtin/v10" v11 "github.com/filecoin-project/go-state-types/builtin/v11" v12 "github.com/filecoin-project/go-state-types/builtin/v12" + v13 "github.com/filecoin-project/go-state-types/builtin/v13" + market13 "github.com/filecoin-project/go-state-types/builtin/v13/market" + adt13 "github.com/filecoin-project/go-state-types/builtin/v13/util/adt" market8 "github.com/filecoin-project/go-state-types/builtin/v8/market" adt8 "github.com/filecoin-project/go-state-types/builtin/v8/util/adt" v9 "github.com/filecoin-project/go-state-types/builtin/v9" @@ -53,6 +67,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/must" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) @@ -72,6 +87,9 @@ var migrationsCmd = &cli.Command{ &cli.BoolFlag{ Name: "check-invariants", }, + &cli.StringFlag{ + Name: "export-bad-migration", + }, }, Action: func(cctx *cli.Context) error { fmt.Println("REMINDER: If you are running this, you likely want to ALSO run the continuity testing tool!") @@ -215,6 +233,31 @@ var migrationsCmd = &cli.Command{ cachedMigrationTime := time.Since(startTime) if newCid1 != newCid2 { + { + if err := printStateDiff(ctx, network.Version(nv), newCid2, newCid1, bs); err != nil { + fmt.Println("failed to print state diff: ", err) + } + } + + if cctx.IsSet("export-bad-migration") { + fi, err := os.Create(cctx.String("export-bad-migration")) + if err != nil { + return xerrors.Errorf("opening the output file: %w", err) + } + + defer fi.Close() //nolint:errcheck + + roots := []cid.Cid{newCid1, newCid2} + + dag := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) + err = car.WriteCarWithWalker(ctx, dag, roots, fi, carWalkFunc) + if err != nil { + return err + } + + fmt.Println("exported bad migration to ", cctx.String("export-bad-migration")) + } + return xerrors.Errorf("got different results with and without the cache: %s, %s", newCid1, newCid2) } @@ -246,6 +289,8 @@ func getMigrationFuncsForNetwork(nv network.Version) (UpgradeActorsFunc, PreUpgr return filcns.UpgradeActorsV11, filcns.PreUpgradeActorsV11, checkNv19Invariants, nil case network.Version21: return filcns.UpgradeActorsV12, filcns.PreUpgradeActorsV12, checkNv21Invariants, nil + case network.Version22: + return filcns.UpgradeActorsV13, filcns.PreUpgradeActorsV13, checkNv22Invariants, nil default: return nil, nil, nil, xerrors.Errorf("migration not implemented for nv%d", nv) } @@ -255,6 +300,357 @@ type UpgradeActorsFunc = func(context.Context, *stmgr.StateManager, stmgr.Migrat type PreUpgradeActorsFunc = func(context.Context, *stmgr.StateManager, stmgr.MigrationCache, cid.Cid, abi.ChainEpoch, *types.TipSet) error type CheckInvariantsFunc = func(context.Context, cid.Cid, cid.Cid, blockstore.Blockstore, abi.ChainEpoch) error +func printStateDiff(ctx context.Context, nv network.Version, newCid1, newCid2 cid.Cid, bs blockstore.Blockstore) error { + // migration diff + var sra, srb types.StateRoot + cst := cbornode.NewCborStore(bs) + + if err := cst.Get(ctx, newCid1, &sra); err != nil { + return err + } + if err := cst.Get(ctx, newCid2, &srb); err != nil { + return err + } + + if sra.Version != srb.Version { + fmt.Println("state root versions do not match: ", sra.Version, srb.Version) + } + if sra.Info != srb.Info { + fmt.Println("state root infos do not match: ", sra.Info, srb.Info) + } + if sra.Actors != srb.Actors { + fmt.Println("state root actors do not match: ", sra.Actors, srb.Actors) + if err := printActorsDiff(ctx, cst, nv, sra.Actors, srb.Actors); err != nil { + return err + } + } + + return nil +} + +func printActorsDiff(ctx context.Context, cst *cbornode.BasicIpldStore, nv network.Version, a, b cid.Cid) error { + // actor diff, a b are a hamt + + diffs, err := hamt.Diff(ctx, cst, cst, a, b, hamt.UseTreeBitWidth(builtin.DefaultHamtBitwidth)) + if err != nil { + return err + } + + keyParser := func(k string) (interface{}, error) { + return address.NewFromBytes([]byte(k)) + } + + for _, d := range diffs { + switch d.Type { + case hamt.Add: + color.Green("+ Add %v", must.One(keyParser(d.Key))) + case hamt.Remove: + color.Red("- Remove %v", must.One(keyParser(d.Key))) + case hamt.Modify: + addr := must.One(keyParser(d.Key)).(address.Address) + color.Yellow("~ Modify %v", addr) + var aa, bb types.ActorV5 + + if err := aa.UnmarshalCBOR(bytes.NewReader(d.Before.Raw)); err != nil { + return err + } + if err := bb.UnmarshalCBOR(bytes.NewReader(d.After.Raw)); err != nil { + return err + } + + if err := printActorDiff(ctx, cst, nv, addr, aa, bb); err != nil { + return err + } + } + } + + return nil +} + +func printActorDiff(ctx context.Context, cst *cbornode.BasicIpldStore, nv network.Version, addr address.Address, a, b types.ActorV5) error { + if a.Code != b.Code { + fmt.Println(" Code: ", a.Code, b.Code) + } + if a.Head != b.Head { + fmt.Println(" Head: ", a.Head, b.Head) + } + if a.Nonce != b.Nonce { + fmt.Println(" Nonce: ", a.Nonce, b.Nonce) + } + if big.Cmp(a.Balance, b.Balance) == 0 { + fmt.Println(" Balance: ", a.Balance, b.Balance) + } + + switch addr.String() { + case "f05": + if err := printMarketActorDiff(ctx, cst, nv, a.Head, b.Head); err != nil { + return err + } + default: + fmt.Println("no logic to diff actor state for ", addr) + } + + return nil +} + +func printMarketActorDiff(ctx context.Context, cst *cbornode.BasicIpldStore, nv network.Version, a, b cid.Cid) error { + if nv != network.Version22 { + return xerrors.Errorf("market actor diff not implemented for nv%d", nv) + } + + var ma, mb market13.State + if err := cst.Get(ctx, a, &ma); err != nil { + return err + } + if err := cst.Get(ctx, b, &mb); err != nil { + return err + } + + if ma.Proposals != mb.Proposals { + fmt.Println(" Proposals: ", ma.Proposals, mb.Proposals) + } + if ma.States != mb.States { + fmt.Println(" States: ", ma.States, mb.States) + + // diff the AMTs + amtDiff, err := amt.Diff(ctx, cst, cst, ma.States, mb.States, amt.UseTreeBitWidth(market13.StatesAmtBitwidth)) + if err != nil { + return err + } + + proposalsArrA, err := adt13.AsArray(adt8.WrapStore(ctx, cst), ma.Proposals, market13.ProposalsAmtBitwidth) + if err != nil { + return err + } + proposalsArrB, err := adt13.AsArray(adt8.WrapStore(ctx, cst), mb.Proposals, market13.ProposalsAmtBitwidth) + if err != nil { + return err + } + + for _, d := range amtDiff { + switch d.Type { + case amt.Add: + color.Green(" state + Add %v", d.Key) + case amt.Remove: + color.Red(" state - Remove %v", d.Key) + case amt.Modify: + color.Yellow(" state ~ Modify %v", d.Key) + + var a, b market13.DealState + if err := a.UnmarshalCBOR(bytes.NewReader(d.Before.Raw)); err != nil { + return err + } + if err := b.UnmarshalCBOR(bytes.NewReader(d.After.Raw)); err != nil { + return err + } + + ja, err := json.Marshal(a) + if err != nil { + return err + } + jb, err := json.Marshal(b) + if err != nil { + return err + } + + fmt.Println(" A: ", string(ja)) + fmt.Println(" B: ", string(jb)) + + var propA, propB market13.DealProposal + + if _, err := proposalsArrA.Get(d.Key, &propA); err != nil { + return err + } + if _, err := proposalsArrB.Get(d.Key, &propB); err != nil { + return err + } + + pab, err := json.Marshal(propA) + if err != nil { + return err + } + pbb, err := json.Marshal(propB) + if err != nil { + return err + } + if string(pab) != string(pbb) { + fmt.Println(" PropA: ", string(pab)) + fmt.Println(" PropB: ", string(pbb)) + } else { + fmt.Println(" Prop: ", string(pab)) + } + + } + } + } + if ma.PendingProposals != mb.PendingProposals { + fmt.Println(" PendingProposals: ", ma.PendingProposals, mb.PendingProposals) + } + if ma.EscrowTable != mb.EscrowTable { + fmt.Println(" EscrowTable: ", ma.EscrowTable, mb.EscrowTable) + } + if ma.LockedTable != mb.LockedTable { + fmt.Println(" LockedTable: ", ma.LockedTable, mb.LockedTable) + } + if ma.NextID != mb.NextID { + fmt.Println(" NextID: ", ma.NextID, mb.NextID) + } + if ma.DealOpsByEpoch != mb.DealOpsByEpoch { + fmt.Println(" DealOpsByEpoch: ", ma.DealOpsByEpoch, mb.DealOpsByEpoch) + } + if ma.LastCron != mb.LastCron { + fmt.Println(" LastCron: ", ma.LastCron, mb.LastCron) + } + if ma.TotalClientLockedCollateral != mb.TotalClientLockedCollateral { + fmt.Println(" TotalClientLockedCollateral: ", ma.TotalClientLockedCollateral, mb.TotalClientLockedCollateral) + } + if ma.TotalProviderLockedCollateral != mb.TotalProviderLockedCollateral { + fmt.Println(" TotalProviderLockedCollateral: ", ma.TotalProviderLockedCollateral, mb.TotalProviderLockedCollateral) + } + if ma.TotalClientStorageFee != mb.TotalClientStorageFee { + fmt.Println(" TotalClientStorageFee: ", ma.TotalClientStorageFee, mb.TotalClientStorageFee) + } + if ma.PendingDealAllocationIds != mb.PendingDealAllocationIds { + fmt.Println(" PendingDealAllocationIds: ", ma.PendingDealAllocationIds, mb.PendingDealAllocationIds) + } + if ma.ProviderSectors != mb.ProviderSectors { + fmt.Println(" ProviderSectors: ", ma.ProviderSectors, mb.ProviderSectors) + + // diff the HAMTs + hamtDiff, err := hamt.Diff(ctx, cst, cst, ma.ProviderSectors, mb.ProviderSectors, hamt.UseTreeBitWidth(market13.ProviderSectorsHamtBitwidth)) + if err != nil { + return err + } + + for _, d := range hamtDiff { + spIDk := must.One(abi.ParseUIntKey(d.Key)) + + switch d.Type { + case hamt.Add: + color.Green(" ProviderSectors + Add f0%v", spIDk) + + var b cbg.CborCid + if err := b.UnmarshalCBOR(bytes.NewReader(d.After.Raw)); err != nil { + return err + } + + fmt.Println(" |-B: ", cid.Cid(b).String()) + + inner, err := adt13.AsMap(adt8.WrapStore(ctx, cst), cid.Cid(b), market13.ProviderSectorsHamtBitwidth) + if err != nil { + return err + } + + var ids market13.SectorDealIDs + err = inner.ForEach(&ids, func(k string) error { + sectorNumber := must.One(abi.ParseUIntKey(k)) + + color.Green(" |-- ProviderSectors + Add %v", sectorNumber) + fmt.Printf(" |+: %v\n", ids) + + return nil + }) + if err != nil { + return err + } + case hamt.Remove: + color.Red(" ProviderSectors - Remove f0%v", spIDk) + case hamt.Modify: + color.Yellow(" ProviderSectors ~ Modify f0%v", spIDk) + + var a, b cbg.CborCid + if err := a.UnmarshalCBOR(bytes.NewReader(d.Before.Raw)); err != nil { + return err + } + if err := b.UnmarshalCBOR(bytes.NewReader(d.After.Raw)); err != nil { + return err + } + + fmt.Println(" |-A: ", cid.Cid(b).String()) + fmt.Println(" |-B: ", cid.Cid(a).String()) + + // diff the inner HAMTs + innerHamtDiff, err := hamt.Diff(ctx, cst, cst, cid.Cid(a), cid.Cid(b), hamt.UseTreeBitWidth(market13.ProviderSectorsHamtBitwidth)) + if err != nil { + return err + } + + for _, d := range innerHamtDiff { + sectorNumber := must.One(abi.ParseUIntKey(d.Key)) + + switch d.Type { + case hamt.Add: + var b market13.SectorDealIDs + + if err := b.UnmarshalCBOR(bytes.NewReader(d.After.Raw)); err != nil { + return err + } + + color.Green(" |-- ProviderSectors + Add %v", sectorNumber) + fmt.Printf(" |B: %v\n", b) + case hamt.Remove: + var a market13.SectorDealIDs + + if err := a.UnmarshalCBOR(bytes.NewReader(d.Before.Raw)); err != nil { + return err + } + + color.Red(" |-- ProviderSectors - Remove %v", sectorNumber) + fmt.Printf(" |A: %v\n", a) + case hamt.Modify: + var a, b market13.SectorDealIDs + + if err := a.UnmarshalCBOR(bytes.NewReader(d.Before.Raw)); err != nil { + return err + } + if err := b.UnmarshalCBOR(bytes.NewReader(d.After.Raw)); err != nil { + return err + } + + color.Yellow(" |-- ProviderSectors ~ Modify %v", sectorNumber) + fmt.Printf(" |A: %v\n", a) + fmt.Printf(" |B: %v\n", b) + } + } + } + } + } + + return nil +} + +func checkNv22Invariants(ctx context.Context, oldStateRootCid cid.Cid, newStateRootCid cid.Cid, bs blockstore.Blockstore, epoch abi.ChainEpoch) error { + + actorStore := store.ActorStore(ctx, bs) + startTime := time.Now() + + // Load the new state root. + var newStateRoot types.StateRoot + if err := actorStore.Get(ctx, newStateRootCid, &newStateRoot); err != nil { + return xerrors.Errorf("failed to decode state root: %w", err) + } + + actorCodeCids, err := actors.GetActorCodeIDs(actorstypes.Version13) + if err != nil { + return err + } + newActorTree, err := builtin.LoadTree(actorStore, newStateRoot.Actors) + if err != nil { + return err + } + messages, err := v13.CheckStateInvariants(newActorTree, epoch, actorCodeCids) + if err != nil { + return xerrors.Errorf("checking state invariants: %w", err) + } + + for _, message := range messages.Messages() { + fmt.Println("got the following error: ", message) + } + + fmt.Println("completed invariant checks, took ", time.Since(startTime)) + + return nil +} func checkNv21Invariants(ctx context.Context, oldStateRootCid cid.Cid, newStateRootCid cid.Cid, bs blockstore.Blockstore, epoch abi.ChainEpoch) error { actorStore := store.ActorStore(ctx, bs) diff --git a/go.mod b/go.mod index 5a86aff1e0c..4bac8d84d28 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/filecoin-project/go-fil-commcid v0.1.0 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 github.com/filecoin-project/go-fil-markets v1.28.3 + github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0 github.com/filecoin-project/go-jsonrpc v0.3.1 github.com/filecoin-project/go-padreader v0.0.1 github.com/filecoin-project/go-paramfetch v0.0.4 @@ -207,7 +208,6 @@ require ( github.com/filecoin-project/go-ds-versioning v0.1.2 // indirect github.com/filecoin-project/go-hamt-ipld v0.1.5 // indirect github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 // indirect - github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0 // indirect github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/gdamore/encoding v1.0.0 // indirect