feat: curio: sectors UI

.gitignore

@@ -39,6 +39,7 @@ build/paramfetch.sh
 *.snap
 devgen.car
 localnet.json
+/*.ndjson
 
 *-fuzz.zip
 /chain/types/work_msg/

chain/types/fil.go

@@ -6,6 +6,8 @@ import (
 	"math/big"
 	"strings"
 
+	"github.com/invopop/jsonschema"
+
 	"github.com/filecoin-project/lotus/build"
 )
 
@@ -138,5 +140,12 @@ func MustParseFIL(s string) FIL {
 	return n
 }
 
+func (f FIL) JSONSchema() *jsonschema.Schema {
+	return &jsonschema.Schema{
+		Type:    "string",
+		Pattern: `^((\d+(\.\d+)?|0x[0-9a-fA-F]+))( ([aA]([tT][tT][oO])?)?[fF][iI][lL])?$`,
+	}
+}
+
 var _ encoding.TextMarshaler = (*FIL)(nil)
 var _ encoding.TextUnmarshaler = (*FIL)(nil)

cli/spcli/sectors.go

@@ -2,6 +2,7 @@ package spcli
 
 import (
 	"bufio"
+	"context"
 	"encoding/csv"
 	"encoding/json"
 	"errors"
@@ -16,6 +17,7 @@ import (
 
 	"github.com/fatih/color"
 	cbor "github.com/ipfs/go-ipld-cbor"
+	"github.com/samber/lo"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
@@ -1353,44 +1355,20 @@ func TerminateSectorCmd(getActorAddress ActorAddressGetter) *cli.Command {
 				}
 			}
 
-			mi, err := nodeApi.StateMinerInfo(ctx, maddr, types.EmptyTSK)
-			if err != nil {
-				return err
-			}
-
-			terminationDeclarationParams := []miner2.TerminationDeclaration{}
-
-			for _, sn := range cctx.Args().Slice() {
-				sectorNum, err := strconv.ParseUint(sn, 10, 64)
+			var outerErr error
+			sectorNumbers := lo.Map(cctx.Args().Slice(), func(sn string, _ int) int {
+				sectorNum, err := strconv.Atoi(sn)
 				if err != nil {
-					return fmt.Errorf("could not parse sector number: %w", err)
+					outerErr = fmt.Errorf("could not parse sector number: %w", err)
+					return 0
 				}
-
-				sectorbit := bitfield.New()
-				sectorbit.Set(sectorNum)
-
-				loca, err := nodeApi.StateSectorPartition(ctx, maddr, abi.SectorNumber(sectorNum), types.EmptyTSK)
-				if err != nil {
-					return fmt.Errorf("get state sector partition %s", err)
-				}
-
-				para := miner2.TerminationDeclaration{
-					Deadline:  loca.Deadline,
-					Partition: loca.Partition,
-					Sectors:   sectorbit,
-				}
-
-				terminationDeclarationParams = append(terminationDeclarationParams, para)
-			}
-
-			terminateSectorParams := &miner2.TerminateSectorsParams{
-				Terminations: terminationDeclarationParams,
+				return sectorNum
+			})
+			if outerErr != nil {
+				return outerErr
 			}
 
-			sp, err := actors.SerializeParams(terminateSectorParams)
-			if err != nil {
-				return xerrors.Errorf("serializing params: %w", err)
-			}
+			confidence := uint64(cctx.Int("confidence"))
 
 			var fromAddr address.Address
 			if from := cctx.String("from"); from != "" {
@@ -1400,33 +1378,81 @@ func TerminateSectorCmd(getActorAddress ActorAddressGetter) *cli.Command {
 					return fmt.Errorf("parsing address %s: %w", from, err)
 				}
 			} else {
+				mi, err := nodeApi.StateMinerInfo(ctx, maddr, types.EmptyTSK)
+				if err != nil {
+					return err
+				}
+
 				fromAddr = mi.Worker
 			}
-
-			smsg, err := nodeApi.MpoolPushMessage(ctx, &types.Message{
-				From:   fromAddr,
-				To:     maddr,
-				Method: builtin.MethodsMiner.TerminateSectors,
-
-				Value:  big.Zero(),
-				Params: sp,
-			}, nil)
+			smsg, err := TerminateSectors(ctx, nodeApi, maddr, sectorNumbers, fromAddr)
 			if err != nil {
-				return xerrors.Errorf("mpool push message: %w", err)
+				return err
 			}
 
-			fmt.Println("sent termination message:", smsg.Cid())
-
-			wait, err := nodeApi.StateWaitMsg(ctx, smsg.Cid(), uint64(cctx.Int("confidence")))
+			wait, err := nodeApi.StateWaitMsg(ctx, smsg.Cid(), confidence)
 			if err != nil {
 				return err
 			}
 
 			if wait.Receipt.ExitCode.IsError() {
 				return fmt.Errorf("terminate sectors message returned exit %d", wait.Receipt.ExitCode)
 			}
-
 			return nil
 		},
 	}
 }
+
+type TerminatorNode interface {
+	StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error)
+	MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error)
+}
+
+func TerminateSectors(ctx context.Context, full TerminatorNode, maddr address.Address, sectorNumbers []int, fromAddr address.Address) (*types.SignedMessage, error) {
+
+	terminationDeclarationParams := []miner2.TerminationDeclaration{}
+
+	for _, sectorNum := range sectorNumbers {
+
+		sectorbit := bitfield.New()
+		sectorbit.Set(uint64(sectorNum))
+
+		loca, err := full.StateSectorPartition(ctx, maddr, abi.SectorNumber(sectorNum), types.EmptyTSK)
+		if err != nil {
+			return nil, fmt.Errorf("get state sector partition %s", err)
+		}
+
+		para := miner2.TerminationDeclaration{
+			Deadline:  loca.Deadline,
+			Partition: loca.Partition,
+			Sectors:   sectorbit,
+		}
+
+		terminationDeclarationParams = append(terminationDeclarationParams, para)
+	}
+
+	terminateSectorParams := &miner2.TerminateSectorsParams{
+		Terminations: terminationDeclarationParams,
+	}
+
+	sp, errA := actors.SerializeParams(terminateSectorParams)
+	if errA != nil {
+		return nil, xerrors.Errorf("serializing params: %w", errA)
+	}
+
+	smsg, err := full.MpoolPushMessage(ctx, &types.Message{
+		From:   fromAddr,
+		To:     maddr,
+		Method: builtin.MethodsMiner.TerminateSectors,
+
+		Value:  big.Zero(),
+		Params: sp,
+	}, nil)
+	if err != nil {
+		return nil, xerrors.Errorf("mpool push message: %w", err)
+	}
+
+	fmt.Println("sent termination message:", smsg.Cid())
+
+	return smsg, nil
+}

cmd/curio/cli.go

@@ -38,6 +38,9 @@ var cliCmd = &cli.Command{
 			// set already
 			return nil
 		}
+		if os.Getenv("LOTUS_DOCS_GENERATION") == "1" {
+			return nil
+		}
 
 		db, err := deps.MakeDB(cctx)
 		if err != nil {

cmd/curio/deps/deps.go

@@ -165,6 +165,7 @@ func GetDeps(ctx context.Context, cctx *cli.Context) (*Deps, error) {
 }
 
 type Deps struct {
+	Layers     []string
 	Cfg        *config.CurioConfig // values
 	DB         *harmonydb.DB       // has itest capability
 	Full       api.FullNode
@@ -212,6 +213,9 @@ func (deps *Deps) PopulateRemainingDeps(ctx context.Context, cctx *cli.Context,
 			return err
 		}
 	}
+	if deps.Layers == nil {
+		deps.Layers = append([]string{"base"}, cctx.StringSlice("layers")...) // Always stack on top of "base" layer
+	}
 
 	if deps.Cfg == nil {
 		// The config feeds into task runners & their helpers

cmd/curio/guidedsetup/guidedsetup.go

@@ -244,7 +244,7 @@ type MigrationData struct {
 
 func complete(d *MigrationData) {
 	stepCompleted(d, d.T("Lotus-Miner to Curio Migration."))
-	d.say(plain, "Try the web interface with %s for further guided improvements.", "--layers=gui")
+	d.say(plain, "Try the web interface with %s for further guided improvements.", code.Render("curio run --layers=gui"))
 	d.say(plain, "You can now migrate your market node (%s), if applicable.", "Boost")
 }
 
@@ -279,7 +279,7 @@ func configToDB(d *MigrationData) {
 
 	chainApiInfo := fmt.Sprintf("%s:%s", string(token), ainfo.Addr)
 
-	d.MinerID, err = SaveConfigToLayer(d.MinerConfigPath, "", false, chainApiInfo)
+	d.MinerID, err = SaveConfigToLayer(d.MinerConfigPath, chainApiInfo)
 	if err != nil {
 		d.say(notice, "Error saving config to layer: %s. Aborting Migration", err.Error())
 		os.Exit(1)
@@ -423,7 +423,7 @@ func verifySectors(d *MigrationData) {
 		time.Sleep(5 * time.Second)
 	}
 	d.say(plain, "The sectors are in the database. The database is ready for %s.", "Curio")
-	d.say(notice, "Now shut down lotus-miner and move the systems to %s.", "Curio")
+	d.say(notice, "Now shut down lotus-miner and lotus-worker and use run %s instead.", code.Render("curio run"))
 
 	_, err = (&promptui.Prompt{Label: d.T("Press return to continue")}).Run()
 	if err != nil {

cmd/curio/guidedsetup/shared.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
-	"errors"
 	"fmt"
 	"os"
 	"path"
@@ -30,7 +29,7 @@ const (
 
 const FlagMinerRepoDeprecation = "storagerepo"
 
-func SaveConfigToLayer(minerRepoPath, layerName string, overwrite bool, chainApiInfo string) (minerAddress address.Address, err error) {
+func SaveConfigToLayer(minerRepoPath, chainApiInfo string) (minerAddress address.Address, err error) {
 	_, say := SetupLanguage()
 	ctx := context.Background()
 
@@ -165,6 +164,12 @@ func SaveConfigToLayer(minerRepoPath, layerName string, overwrite bool, chainApi
 			baseCfg.Addresses = lo.Filter(baseCfg.Addresses, func(a config.CurioAddresses, _ int) bool {
 				return len(a.MinerAddresses) > 0
 			})
+			if baseCfg.Apis.ChainApiInfo == nil {
+				baseCfg.Apis.ChainApiInfo = append(baseCfg.Apis.ChainApiInfo, chainApiInfo)
+			}
+			if baseCfg.Apis.StorageRPCSecret == "" {
+				baseCfg.Apis.StorageRPCSecret = curioCfg.Apis.StorageRPCSecret
+			}
 
 			cb, err := config.ConfigUpdate(baseCfg, config.DefaultCurioConfig(), config.Commented(true), config.DefaultKeepUncommented(), config.NoEnv())
 			if err != nil {
@@ -174,44 +179,33 @@ func SaveConfigToLayer(minerRepoPath, layerName string, overwrite bool, chainApi
 			if err != nil {
 				return minerAddress, xerrors.Errorf("cannot update base config: %w", err)
 			}
-			say(plain, "Configuration 'base' was updated to include this miner's address and its wallet setup.")
+			say(plain, "Configuration 'base' was updated to include this miner's address (%s) and its wallet setup.", minerAddress)
 		}
 		say(plain, "Compare the configurations %s to %s. Changes between the miner IDs other than wallet addreses should be a new, minimal layer for runners that need it.", "base", "mig-"+curioCfg.Addresses[0].MinerAddresses[0])
 	skipWritingToBase:
-	} else if layerName == "" {
-		cfg, err := deps.GetDefaultConfig(true)
-		if err != nil {
-			return minerAddress, xerrors.Errorf("Cannot get default config: %w", err)
-		}
+	} else {
 		_, err = db.Exec(ctx, `INSERT INTO harmony_config (title, config) VALUES ('base', $1)
-		 ON CONFLICT(title) DO UPDATE SET config=EXCLUDED.config`, cfg)
+		 ON CONFLICT(title) DO UPDATE SET config=EXCLUDED.config`, configTOML)
 
 		if err != nil {
 			return minerAddress, xerrors.Errorf("Cannot insert base config: %w", err)
 		}
-		say(notice, "Configuration 'base' was created to include this miner's address and its wallet setup.")
+		say(notice, "Configuration 'base' was created to resemble this lotus-miner's config.toml .")
 	}
 
-	if layerName == "" { // only make mig if base exists and we are different. // compare to base.
-		layerName = fmt.Sprintf("mig-%s", curioCfg.Addresses[0].MinerAddresses[0])
-		overwrite = true
-	} else {
-		if lo.Contains(titles, layerName) && !overwrite {
-			return minerAddress, errors.New("the overwrite flag is needed to replace existing layer: " + layerName)
-		}
-	}
-	if overwrite {
-		_, err := db.Exec(ctx, "DELETE FROM harmony_config WHERE title=$1", layerName)
+	{ // make a layer representing the migration
+		layerName := fmt.Sprintf("mig-%s", curioCfg.Addresses[0].MinerAddresses[0])
+		_, err = db.Exec(ctx, "DELETE FROM harmony_config WHERE title=$1", layerName)
 		if err != nil {
 			return minerAddress, xerrors.Errorf("Cannot delete existing layer: %w", err)
 		}
-	}
 
-	_, err = db.Exec(ctx, "INSERT INTO harmony_config (title, config) VALUES ($1, $2)", layerName, configTOML.String())
-	if err != nil {
-		return minerAddress, xerrors.Errorf("Cannot insert layer after layer created message: %w", err)
+		_, err = db.Exec(ctx, "INSERT INTO harmony_config (title, config) VALUES ($1, $2)", layerName, configTOML.String())
+		if err != nil {
+			return minerAddress, xerrors.Errorf("Cannot insert layer after layer created message: %w", err)
+		}
+		say(plain, "Layer %s created. ", layerName)
 	}
-	say(plain, "Layer %s created. ", layerName)
 
 	dbSettings := getDBSettings(*smCfg)
 	say(plain, "To work with the config: ")