From 02f3c9d46cf019f8ad00ecce2e902c152daf018a Mon Sep 17 00:00:00 2001 From: Aryan Tikarya Date: Fri, 22 Nov 2024 14:56:57 +0530 Subject: [PATCH 01/15] refactor the commands to generate the cli docs --- {cmd/lotus-worker => cli/worker}/info.go | 2 +- {cmd/lotus-worker => cli/worker}/resources.go | 2 +- cmd/lotus-worker/cli.go => cli/worker/set.go | 2 +- {cmd/lotus-worker => cli/worker}/storage.go | 2 +- {cmd/lotus-worker => cli/worker}/tasks.go | 2 +- cli/worker/worker.go | 873 ++++++++++++++++++ cmd/lotus-worker/main.go | 864 +---------------- 7 files changed, 881 insertions(+), 866 deletions(-) rename {cmd/lotus-worker => cli/worker}/info.go (99%) rename {cmd/lotus-worker => cli/worker}/resources.go (98%) rename cmd/lotus-worker/cli.go => cli/worker/set.go (98%) rename {cmd/lotus-worker => cli/worker}/storage.go (99%) rename {cmd/lotus-worker => cli/worker}/tasks.go (99%) create mode 100644 cli/worker/worker.go diff --git a/cmd/lotus-worker/info.go b/cli/worker/info.go similarity index 99% rename from cmd/lotus-worker/info.go rename to cli/worker/info.go index 338345c2857..a8197141928 100644 --- a/cmd/lotus-worker/info.go +++ b/cli/worker/info.go @@ -1,4 +1,4 @@ -package main +package worker import ( "fmt" diff --git a/cmd/lotus-worker/resources.go b/cli/worker/resources.go similarity index 98% rename from cmd/lotus-worker/resources.go rename to cli/worker/resources.go index 45fd01e702b..6d79f9523be 100644 --- a/cmd/lotus-worker/resources.go +++ b/cli/worker/resources.go @@ -1,4 +1,4 @@ -package main +package worker import ( "fmt" diff --git a/cmd/lotus-worker/cli.go b/cli/worker/set.go similarity index 98% rename from cmd/lotus-worker/cli.go rename to cli/worker/set.go index 393ece18c86..bdbe15ef71c 100644 --- a/cmd/lotus-worker/cli.go +++ b/cli/worker/set.go @@ -1,4 +1,4 @@ -package main +package worker import ( "fmt" diff --git a/cmd/lotus-worker/storage.go b/cli/worker/storage.go similarity index 99% rename from cmd/lotus-worker/storage.go rename to cli/worker/storage.go index 8da35c94418..42e7b5ae3c1 100644 --- a/cmd/lotus-worker/storage.go +++ b/cli/worker/storage.go @@ -1,4 +1,4 @@ -package main +package worker import ( "encoding/json" diff --git a/cmd/lotus-worker/tasks.go b/cli/worker/tasks.go similarity index 99% rename from cmd/lotus-worker/tasks.go rename to cli/worker/tasks.go index 73ca5eb0a01..a67a53f0221 100644 --- a/cmd/lotus-worker/tasks.go +++ b/cli/worker/tasks.go @@ -1,4 +1,4 @@ -package main +package worker import ( "context" diff --git a/cli/worker/worker.go b/cli/worker/worker.go new file mode 100644 index 00000000000..3255d55bd17 --- /dev/null +++ b/cli/worker/worker.go @@ -0,0 +1,873 @@ +package worker + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "path/filepath" + "reflect" + "strings" + "time" + + "github.com/google/uuid" + "github.com/ipfs/go-datastore/namespace" + logging "github.com/ipfs/go-log/v2" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" + "github.com/urfave/cli/v2" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-jsonrpc/auth" + "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/go-statestore" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/build/buildconstants" + lcli "github.com/filecoin-project/lotus/cli" + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker" + "github.com/filecoin-project/lotus/lib/ulimit" + "github.com/filecoin-project/lotus/metrics" + "github.com/filecoin-project/lotus/node/modules" + "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" +) + +const FlagWorkerRepo = "worker-repo" + +// TODO remove after deprecation period +const FlagWorkerRepoDeprecation = "workerrepo" + +var log = logging.Logger("worker") + +func App() *cli.App { + local := []*cli.Command{ + runCmd, + stopCmd, + infoCmd, + storageCmd, + setCmd, + waitQuietCmd, + resourcesCmd, + tasksCmd, + } + + app := &cli.App{ + Name: "lotus-worker", + Usage: "Remote miner worker", + Version: string(build.MinerUserVersion()), + EnableBashCompletion: true, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: FlagWorkerRepo, + Aliases: []string{FlagWorkerRepoDeprecation}, + EnvVars: []string{"LOTUS_WORKER_PATH", "WORKER_PATH"}, + Value: "~/.lotusworker", // TODO: Consider XDG_DATA_HOME + Usage: fmt.Sprintf("Specify worker repo path. flag %s and env WORKER_PATH are DEPRECATION, will REMOVE SOON", FlagWorkerRepoDeprecation), + }, + &cli.StringFlag{ + Name: "panic-reports", + EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, + Hidden: true, + Value: "~/.lotusworker", // should follow --repo default + }, + &cli.StringFlag{ + Name: "miner-repo", + Aliases: []string{"storagerepo"}, + EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"}, + Value: "~/.lotusminer", // TODO: Consider XDG_DATA_HOME + Usage: fmt.Sprintf("Specify miner repo path. flag storagerepo and env LOTUS_STORAGE_PATH are DEPRECATION, will REMOVE SOON"), + }, + &cli.BoolFlag{ + Name: "enable-gpu-proving", + Usage: "enable use of GPU for mining operations", + Value: true, + EnvVars: []string{"LOTUS_WORKER_ENABLE_GPU_PROVING"}, + }, + }, + + After: func(c *cli.Context) error { + if r := recover(); r != nil { + // Generate report in LOTUS_PANIC_REPORT_PATH and re-raise panic + build.GenerateMinerPanicReport(c.String("panic-reports"), c.String(FlagWorkerRepo), c.App.Name) + panic(r) + } + return nil + }, + Commands: local, + } + + app.Setup() + app.Metadata["repoType"] = repo.Worker + return app +} + +var runCmd = &cli.Command{ + Name: "run", + Usage: "Start lotus worker", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "listen", + Usage: "host address and port the worker api will listen on", + Value: "0.0.0.0:3456", + EnvVars: []string{"LOTUS_WORKER_LISTEN"}, + }, + &cli.StringFlag{ + Name: "address", + Hidden: true, + }, + &cli.BoolFlag{ + Name: "no-local-storage", + Usage: "don't use storageminer repo for sector storage", + EnvVars: []string{"LOTUS_WORKER_NO_LOCAL_STORAGE"}, + }, + &cli.BoolFlag{ + Name: "no-swap", + Usage: "don't use swap", + Value: false, + EnvVars: []string{"LOTUS_WORKER_NO_SWAP"}, + }, + &cli.StringFlag{ + Name: "name", + Usage: "custom worker name", + EnvVars: []string{"LOTUS_WORKER_NAME"}, + DefaultText: "hostname", + }, + &cli.BoolFlag{ + Name: "addpiece", + Usage: "enable addpiece", + Value: true, + EnvVars: []string{"LOTUS_WORKER_ADDPIECE"}, + }, + &cli.BoolFlag{ + Name: "precommit1", + Usage: "enable precommit1", + Value: true, + EnvVars: []string{"LOTUS_WORKER_PRECOMMIT1"}, + }, + &cli.BoolFlag{ + Name: "unseal", + Usage: "enable unsealing", + Value: true, + EnvVars: []string{"LOTUS_WORKER_UNSEAL"}, + }, + &cli.BoolFlag{ + Name: "precommit2", + Usage: "enable precommit2", + Value: true, + EnvVars: []string{"LOTUS_WORKER_PRECOMMIT2"}, + }, + &cli.BoolFlag{ + Name: "commit", + Usage: "enable commit", + Value: true, + EnvVars: []string{"LOTUS_WORKER_COMMIT"}, + }, + &cli.BoolFlag{ + Name: "replica-update", + Usage: "enable replica update", + Value: true, + EnvVars: []string{"LOTUS_WORKER_REPLICA_UPDATE"}, + }, + &cli.BoolFlag{ + Name: "prove-replica-update2", + Usage: "enable prove replica update 2", + Value: true, + EnvVars: []string{"LOTUS_WORKER_PROVE_REPLICA_UPDATE2"}, + }, + &cli.BoolFlag{ + Name: "regen-sector-key", + Usage: "enable regen sector key", + Value: true, + EnvVars: []string{"LOTUS_WORKER_REGEN_SECTOR_KEY"}, + }, + &cli.BoolFlag{ + Name: "sector-download", + Usage: "enable external sector data download", + Value: false, + EnvVars: []string{"LOTUS_WORKER_SECTOR_DOWNLOAD"}, + }, + &cli.BoolFlag{ + Name: "windowpost", + Usage: "enable window post", + Value: false, + EnvVars: []string{"LOTUS_WORKER_WINDOWPOST"}, + }, + &cli.BoolFlag{ + Name: "winningpost", + Usage: "enable winning post", + Value: false, + EnvVars: []string{"LOTUS_WORKER_WINNINGPOST"}, + }, + &cli.BoolFlag{ + Name: "no-default", + Usage: "disable all default compute tasks, use the worker for storage/fetching only", + Value: false, + EnvVars: []string{"LOTUS_WORKER_NO_DEFAULT"}, + }, + &cli.IntFlag{ + Name: "parallel-fetch-limit", + Usage: "maximum fetch operations to run in parallel", + Value: 5, + EnvVars: []string{"LOTUS_WORKER_PARALLEL_FETCH_LIMIT"}, + }, + &cli.IntFlag{ + Name: "post-parallel-reads", + Usage: "maximum number of parallel challenge reads (0 = no limit)", + Value: 32, + EnvVars: []string{"LOTUS_WORKER_POST_PARALLEL_READS"}, + }, + &cli.DurationFlag{ + Name: "post-read-timeout", + Usage: "time limit for reading PoSt challenges (0 = no limit)", + Value: 0, + EnvVars: []string{"LOTUS_WORKER_POST_READ_TIMEOUT"}, + }, + &cli.StringFlag{ + Name: "timeout", + Usage: "used when 'listen' is unspecified. must be a valid duration recognized by golang's time.ParseDuration function", + Value: "30m", + EnvVars: []string{"LOTUS_WORKER_TIMEOUT"}, + }, + &cli.StringFlag{ + Name: "http-server-timeout", + Value: "30s", + }, + &cli.BoolFlag{ + Name: "data-cid", + Usage: "Run the data-cid task. true|false", + Value: true, + DefaultText: "inherits --addpiece", + }, + &cli.StringFlag{ + Name: "external-pc2", + Usage: "command for computing PC2 externally", + }, + }, + Description: `Run lotus-worker. + +--external-pc2 can be used to compute the PreCommit2 inputs externally. +The flag behaves similarly to the related lotus-worker flag, using it in +lotus-bench may be useful for testing if the external PreCommit2 command is +invoked correctly. + +The command will be called with a number of environment variables set: +* EXTSEAL_PC2_SECTOR_NUM: the sector number +* EXTSEAL_PC2_SECTOR_MINER: the miner id +* EXTSEAL_PC2_PROOF_TYPE: the proof type +* EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes +* EXTSEAL_PC2_CACHE: the path to the cache directory +* EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller) +* EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json) + +The command is expected to: +* Create cache sc-02-data-tree-r* files +* Create cache sc-02-data-tree-c* files +* Create cache p_aux / t_aux files +* Transform the sealed file in place + +Example invocation of lotus-bench as external executor: +'./lotus-bench simple precommit2 --sector-size $EXTSEAL_PC2_SECTOR_SIZE $EXTSEAL_PC2_SEALED $EXTSEAL_PC2_CACHE $EXTSEAL_PC2_PC1OUT' +`, + Before: func(cctx *cli.Context) error { + if cctx.IsSet("address") { + log.Warnf("The '--address' flag is deprecated, it has been replaced by '--listen'") + if err := cctx.Set("listen", cctx.String("address")); err != nil { + return err + } + } + + return nil + }, + Action: func(cctx *cli.Context) error { + log.Info("Starting lotus worker") + + if !cctx.Bool("enable-gpu-proving") { + if err := os.Setenv("BELLMAN_NO_GPU", "true"); err != nil { + return xerrors.Errorf("could not set no-gpu env: %+v", err) + } + } + + // ensure tmpdir exists + td := os.TempDir() + if err := os.MkdirAll(td, 0755); err != nil { + return xerrors.Errorf("ensuring temp dir %s exists: %w", td, err) + } + + // Check file descriptor limit + limit, _, err := ulimit.GetLimit() + switch { + case err == ulimit.ErrUnsupported: + log.Errorw("checking file descriptor limit failed", "error", err) + case err != nil: + return xerrors.Errorf("checking fd limit: %w", err) + default: + if limit < buildconstants.MinerFDLimit { + return xerrors.Errorf("soft file descriptor limit (ulimit -n) too low, want %d, current %d", buildconstants.MinerFDLimit, limit) + } + } + + // Check DC-environment variable + sectorSizes := []string{"2KiB", "8MiB", "512MiB", "32GiB", "64GiB"} + resourcesType := reflect.TypeOf(storiface.Resources{}) + + for _, sectorSize := range sectorSizes { + for i := 0; i < resourcesType.NumField(); i++ { + field := resourcesType.Field(i) + envName := field.Tag.Get("envname") + if envName != "" { + // Check if DC_[SectorSize]_[ResourceRestriction] is set + envVar, ok := os.LookupEnv("DC_" + sectorSize + "_" + envName) + if ok { + // If it is set, convert it to DC_[ResourceRestriction] + err := os.Setenv("DC_"+envName, envVar) + if err != nil { + log.Fatalf("Error setting environment variable: %v", err) + } + log.Warnf("Converted DC_%s_%s to DC_%s, because DC is a sector-size independent job", sectorSize, envName, envName) + } + } + } + } + + // Connect to storage-miner + ctx := lcli.ReqContext(cctx) + + // Create a new context with cancel function + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Listen for interrupt signals + go func() { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + <-c + cancel() + }() + + var nodeApi api.StorageMiner + var closer func() + for { + nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp) + if err == nil { + _, err = nodeApi.Version(ctx) + if err == nil { + break + } + } + fmt.Printf("\r\x1b[0KConnecting to miner API... (%s)", err) + select { + case <-ctx.Done(): + return xerrors.New("Interrupted by user") + case <-time.After(time.Second): + } + } + defer closer() + // Register all metric views + if err := view.Register( + metrics.DefaultViews..., + ); err != nil { + log.Fatalf("Cannot register the view: %v", err) + } + + v, err := nodeApi.Version(ctx) + if err != nil { + return err + } + if v.APIVersion != api.MinerAPIVersion0 { + return xerrors.Errorf("lotus-miner API version doesn't match: expected: %s", api.APIVersion{APIVersion: api.MinerAPIVersion0}) + } + log.Infof("Remote version %s", v) + + // Check params + + act, err := nodeApi.ActorAddress(ctx) + if err != nil { + return err + } + ssize, err := nodeApi.ActorSectorSize(ctx, act) + if err != nil { + return err + } + + var taskTypes []sealtasks.TaskType + var workerType string + var needParams bool + + if cctx.Bool("windowpost") { + needParams = true + workerType = sealtasks.WorkerWindowPoSt + taskTypes = append(taskTypes, sealtasks.TTGenerateWindowPoSt) + } + if cctx.Bool("winningpost") { + needParams = true + workerType = sealtasks.WorkerWinningPoSt + taskTypes = append(taskTypes, sealtasks.TTGenerateWinningPoSt) + } + + if workerType == "" { + taskTypes = append(taskTypes, sealtasks.TTFetch, sealtasks.TTCommit1, sealtasks.TTProveReplicaUpdate1, sealtasks.TTFinalize, sealtasks.TTFinalizeUnsealed, sealtasks.TTFinalizeReplicaUpdate) + + if !cctx.Bool("no-default") { + workerType = sealtasks.WorkerSealing + } + } + + ttDataCidDefault := false + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("addpiece")) && cctx.Bool("addpiece") { + taskTypes = append(taskTypes, sealtasks.TTAddPiece) + ttDataCidDefault = true + } + if workerType == sealtasks.WorkerSealing { + if cctx.IsSet("data-cid") { + if cctx.Bool("data-cid") { + taskTypes = append(taskTypes, sealtasks.TTDataCid) + } + } else if ttDataCidDefault { + taskTypes = append(taskTypes, sealtasks.TTDataCid) + } + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("sector-download")) && cctx.Bool("sector-download") { + taskTypes = append(taskTypes, sealtasks.TTDownloadSector) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("precommit1")) && cctx.Bool("precommit1") { + taskTypes = append(taskTypes, sealtasks.TTPreCommit1) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("unseal")) && cctx.Bool("unseal") { + taskTypes = append(taskTypes, sealtasks.TTUnseal) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("precommit2")) && cctx.Bool("precommit2") { + taskTypes = append(taskTypes, sealtasks.TTPreCommit2) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("commit")) && cctx.Bool("commit") { + needParams = true + taskTypes = append(taskTypes, sealtasks.TTCommit2) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("replica-update")) && cctx.Bool("replica-update") { + taskTypes = append(taskTypes, sealtasks.TTReplicaUpdate) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("prove-replica-update2")) && cctx.Bool("prove-replica-update2") { + needParams = true + taskTypes = append(taskTypes, sealtasks.TTProveReplicaUpdate2) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("regen-sector-key")) && cctx.Bool("regen-sector-key") { + taskTypes = append(taskTypes, sealtasks.TTRegenSectorKey) + } + + if cctx.Bool("no-default") && workerType == "" { + workerType = sealtasks.WorkerSealing + } + + if len(taskTypes) == 0 { + return xerrors.Errorf("no task types specified") + } + for _, taskType := range taskTypes { + if taskType.WorkerType() != workerType { + return xerrors.Errorf("expected all task types to be for %s worker, but task %s is for %s worker", workerType, taskType, taskType.WorkerType()) + } + } + + if needParams { + if err := paramfetch.GetParams(ctx, build.ParametersJSON(), build.SrsJSON(), uint64(ssize)); err != nil { + return xerrors.Errorf("get params: %w", err) + } + } + + // Open repo + + repoPath := cctx.String(FlagWorkerRepo) + r, err := repo.NewFS(repoPath) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + if !ok { + if err := r.Init(repo.Worker); err != nil { + return err + } + + lr, err := r.Lock(repo.Worker) + if err != nil { + return err + } + + var localPaths []storiface.LocalPath + + if !cctx.Bool("no-local-storage") { + b, err := json.MarshalIndent(&storiface.LocalStorageMeta{ + ID: storiface.ID(uuid.New().String()), + Weight: 10, + CanSeal: true, + CanStore: false, + }, "", " ") + if err != nil { + return xerrors.Errorf("marshaling storage config: %w", err) + } + + if err := os.WriteFile(filepath.Join(lr.Path(), "sectorstore.json"), b, 0644); err != nil { + return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "sectorstore.json"), err) + } + + localPaths = append(localPaths, storiface.LocalPath{ + Path: lr.Path(), + }) + } + + if err := lr.SetStorage(func(sc *storiface.StorageConfig) { + sc.StoragePaths = append(sc.StoragePaths, localPaths...) + }); err != nil { + return xerrors.Errorf("set storage config: %w", err) + } + + { + // init datastore for r.Exists + _, err := lr.Datastore(context.Background(), "/metadata") + if err != nil { + return err + } + } + if err := lr.Close(); err != nil { + return xerrors.Errorf("close repo: %w", err) + } + } + + lr, err := r.Lock(repo.Worker) + if err != nil { + return err + } + defer func() { + if err := lr.Close(); err != nil { + log.Error("closing repo", err) + } + }() + ds, err := lr.Datastore(context.Background(), "/metadata") + if err != nil { + return err + } + + log.Info("Opening local storage; connecting to master") + const unspecifiedAddress = "0.0.0.0" + + address := cctx.String("listen") + host, port, err := net.SplitHostPort(address) + if err != nil { + return err + } + + if ip := net.ParseIP(host); ip != nil { + if ip.String() == unspecifiedAddress { + timeout, err := time.ParseDuration(cctx.String("timeout")) + if err != nil { + return err + } + rip, err := extractRoutableIP(timeout) + if err != nil { + return err + } + host = rip + } + } + + var newAddress string + + // Check if the IP address is IPv6 + ip := net.ParseIP(host) + if ip.To4() == nil && ip.To16() != nil { + newAddress = "[" + host + "]:" + port + } else { + newAddress = host + ":" + port + } + + localStore, err := paths.NewLocal(ctx, lr, nodeApi, []string{"http://" + newAddress + "/remote"}) + if err != nil { + return err + } + + // Setup remote sector store + sminfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) + if err != nil { + return xerrors.Errorf("could not get api info: %w", err) + } + + remote := paths.NewRemote(localStore, nodeApi, sminfo.AuthHeader(), cctx.Int("parallel-fetch-limit"), + &paths.DefaultPartialFileHandler{}) + + fh := &paths.FetchHandler{Local: localStore, PfHandler: &paths.DefaultPartialFileHandler{}} + remoteHandler := func(w http.ResponseWriter, r *http.Request) { + if !auth.HasPerm(r.Context(), nil, api.PermAdmin) { + w.WriteHeader(401) + _ = json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing admin permission"}) + return + } + + fh.ServeHTTP(w, r) + } + + // Parse ffi executor flags + + var ffiOpts []ffiwrapper.FFIWrapperOpt + + if cctx.IsSet("external-pc2") { + extSeal := ffiwrapper.ExternalSealer{ + PreCommit2: ffiwrapper.MakeExternPrecommit2(cctx.String("external-pc2")), + } + + ffiOpts = append(ffiOpts, ffiwrapper.WithExternalSealCalls(extSeal)) + } + + // Create / expose the worker + + wsts := statestore.New(namespace.Wrap(ds, modules.WorkerCallsPrefix)) + + workerApi := &sealworker.Worker{ + LocalWorker: sealer.NewLocalWorkerWithExecutor( + sealer.FFIExec(ffiOpts...), + sealer.WorkerConfig{ + TaskTypes: taskTypes, + NoSwap: cctx.Bool("no-swap"), + MaxParallelChallengeReads: cctx.Int("post-parallel-reads"), + ChallengeReadTimeout: cctx.Duration("post-read-timeout"), + Name: cctx.String("name"), + }, os.LookupEnv, remote, localStore, nodeApi, nodeApi, wsts), + LocalStore: localStore, + Storage: lr, + } + + log.Info("Setting up control endpoint at " + newAddress) + + timeout, err := time.ParseDuration(cctx.String("http-server-timeout")) + if err != nil { + return xerrors.Errorf("invalid time string %s: %x", cctx.String("http-server-timeout"), err) + } + + srv := &http.Server{ + Handler: sealworker.WorkerHandler(nodeApi.AuthVerify, remoteHandler, workerApi, true), + ReadHeaderTimeout: timeout, + BaseContext: func(listener net.Listener) context.Context { + ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-worker")) + return ctx + }, + } + + go func() { + <-ctx.Done() + log.Warn("Shutting down...") + if err := srv.Shutdown(context.TODO()); err != nil { + log.Errorf("shutting down RPC server failed: %s", err) + } + log.Warn("Graceful shutdown successful") + }() + + nl, err := net.Listen("tcp", newAddress) + if err != nil { + return err + } + + { + a, err := net.ResolveTCPAddr("tcp", newAddress) + if err != nil { + return xerrors.Errorf("parsing address: %w", err) + } + + ma, err := manet.FromNetAddr(a) + if err != nil { + return xerrors.Errorf("creating api multiaddress: %w", err) + } + + if err := lr.SetAPIEndpoint(ma); err != nil { + return xerrors.Errorf("setting api endpoint: %w", err) + } + + ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) + if err != nil { + return xerrors.Errorf("could not get miner API info: %w", err) + } + + // TODO: ideally this would be a token with some permissions dropped + if err := lr.SetAPIToken(ainfo.Token); err != nil { + return xerrors.Errorf("setting api token: %w", err) + } + } + + minerSession, err := nodeApi.Session(ctx) + if err != nil { + return xerrors.Errorf("getting miner session: %w", err) + } + + waitQuietCh := func() chan struct{} { + out := make(chan struct{}) + go func() { + workerApi.LocalWorker.WaitQuiet() + close(out) + }() + return out + } + + go func() { + heartbeats := time.NewTicker(paths.HeartbeatInterval) + defer heartbeats.Stop() + + var redeclareStorage bool + var readyCh chan struct{} + for { + // If we're reconnecting, redeclare storage first + if redeclareStorage { + log.Info("Redeclaring local storage") + + if err := localStore.Redeclare(ctx, nil, false); err != nil { + log.Errorf("Redeclaring local storage failed: %+v", err) + + select { + case <-ctx.Done(): + return // graceful shutdown + case <-heartbeats.C: + } + continue + } + } + + // TODO: we could get rid of this, but that requires tracking resources for restarted tasks correctly + if readyCh == nil { + log.Info("Making sure no local tasks are running") + readyCh = waitQuietCh() + } + + for { + curSession, err := nodeApi.Session(ctx) + if err != nil { + log.Errorf("heartbeat: checking remote session failed: %+v", err) + } else { + if curSession != minerSession { + minerSession = curSession + break + } + } + + select { + case <-readyCh: + if err := nodeApi.WorkerConnect(ctx, "http://"+newAddress+"/rpc/v0"); err != nil { + log.Errorf("Registering worker failed: %+v", err) + cancel() + return + } + + log.Info("Worker registered successfully, waiting for tasks") + + readyCh = nil + case <-heartbeats.C: + case <-ctx.Done(): + return // graceful shutdown + } + } + + log.Errorf("LOTUS-MINER CONNECTION LOST") + + redeclareStorage = true + } + }() + + go func() { + <-workerApi.Done() + // Wait 20s to allow the miner to unregister the worker on next heartbeat + time.Sleep(20 * time.Second) + log.Warn("Shutting down...") + if err := srv.Shutdown(context.TODO()); err != nil { + log.Errorf("shutting down RPC server failed: %s", err) + } + log.Warn("Graceful shutdown successful") + }() + + return srv.Serve(nl) + }, +} + +var stopCmd = &cli.Command{ + Name: "stop", + Usage: "Stop a running lotus worker", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + api, closer, err := lcli.GetWorkerAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := lcli.ReqContext(cctx) + + // Detach any storage associated with this worker + err = api.StorageDetachAll(ctx) + if err != nil { + return err + } + + err = api.Shutdown(ctx) + if err != nil { + return err + } + + return nil + }, +} + +func extractRoutableIP(timeout time.Duration) (string, error) { + minerMultiAddrKey := "MINER_API_INFO" + deprecatedMinerMultiAddrKey := "STORAGE_API_INFO" + env, ok := os.LookupEnv(minerMultiAddrKey) + if !ok { + _, ok = os.LookupEnv(deprecatedMinerMultiAddrKey) + if ok { + log.Warnf("Using a deprecated env(%s) value, please use env(%s) instead.", deprecatedMinerMultiAddrKey, minerMultiAddrKey) + } + return "", xerrors.New("MINER_API_INFO environment variable required to extract IP") + } + + // Splitting the env to separate the JWT from the multiaddress + splitEnv := strings.SplitN(env, ":", 2) + if len(splitEnv) < 2 { + return "", xerrors.Errorf("invalid MINER_API_INFO format") + } + // Only take the multiaddress part + maddrStr := splitEnv[1] + + maddr, err := multiaddr.NewMultiaddr(maddrStr) + if err != nil { + return "", err + } + + minerIP, _ := maddr.ValueForProtocol(multiaddr.P_IP6) + if minerIP == "" { + minerIP, _ = maddr.ValueForProtocol(multiaddr.P_IP4) + } + minerPort, _ := maddr.ValueForProtocol(multiaddr.P_TCP) + + // Format the address appropriately + addressToDial := net.JoinHostPort(minerIP, minerPort) + + conn, err := net.DialTimeout("tcp", addressToDial, timeout) + if err != nil { + return "", err + } + + defer func() { + if cerr := conn.Close(); cerr != nil { + log.Errorf("Error closing connection: %v", cerr) + } + }() + + localAddr := conn.LocalAddr().(*net.TCPAddr) + return localAddr.IP.String(), nil +} diff --git a/cmd/lotus-worker/main.go b/cmd/lotus-worker/main.go index 6d70b8240e8..b67da77f21c 100644 --- a/cmd/lotus-worker/main.go +++ b/cmd/lotus-worker/main.go @@ -1,882 +1,24 @@ package main import ( - "context" - "encoding/json" - "fmt" - "net" - "net/http" "os" - "os/signal" - "path/filepath" - "reflect" - "strings" - "time" - - "github.com/google/uuid" - "github.com/ipfs/go-datastore/namespace" - logging "github.com/ipfs/go-log/v2" - "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" - "github.com/urfave/cli/v2" - "go.opencensus.io/stats/view" - "go.opencensus.io/tag" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-jsonrpc/auth" - "github.com/filecoin-project/go-paramfetch" - "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/build/buildconstants" - lcli "github.com/filecoin-project/lotus/cli" - cliutil "github.com/filecoin-project/lotus/cli/util" - "github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker" + "github.com/filecoin-project/lotus/cli/worker" "github.com/filecoin-project/lotus/lib/lotuslog" - "github.com/filecoin-project/lotus/lib/ulimit" - "github.com/filecoin-project/lotus/metrics" - "github.com/filecoin-project/lotus/node/modules" - "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/storage/paths" - "github.com/filecoin-project/lotus/storage/sealer" - "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" - "github.com/filecoin-project/lotus/storage/sealer/sealtasks" - "github.com/filecoin-project/lotus/storage/sealer/storiface" + logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("main") -const FlagWorkerRepo = "worker-repo" - -// TODO remove after deprecation period -const FlagWorkerRepoDeprecation = "workerrepo" - func main() { api.RunningNodeType = api.NodeWorker lotuslog.SetupLogLevels() - local := []*cli.Command{ - runCmd, - stopCmd, - infoCmd, - storageCmd, - setCmd, - waitQuietCmd, - resourcesCmd, - tasksCmd, - } - - app := &cli.App{ - Name: "lotus-worker", - Usage: "Remote miner worker", - Version: string(build.MinerUserVersion()), - EnableBashCompletion: true, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: FlagWorkerRepo, - Aliases: []string{FlagWorkerRepoDeprecation}, - EnvVars: []string{"LOTUS_WORKER_PATH", "WORKER_PATH"}, - Value: "~/.lotusworker", // TODO: Consider XDG_DATA_HOME - Usage: fmt.Sprintf("Specify worker repo path. flag %s and env WORKER_PATH are DEPRECATION, will REMOVE SOON", FlagWorkerRepoDeprecation), - }, - &cli.StringFlag{ - Name: "panic-reports", - EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, - Hidden: true, - Value: "~/.lotusworker", // should follow --repo default - }, - &cli.StringFlag{ - Name: "miner-repo", - Aliases: []string{"storagerepo"}, - EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"}, - Value: "~/.lotusminer", // TODO: Consider XDG_DATA_HOME - Usage: fmt.Sprintf("Specify miner repo path. flag storagerepo and env LOTUS_STORAGE_PATH are DEPRECATION, will REMOVE SOON"), - }, - &cli.BoolFlag{ - Name: "enable-gpu-proving", - Usage: "enable use of GPU for mining operations", - Value: true, - EnvVars: []string{"LOTUS_WORKER_ENABLE_GPU_PROVING"}, - }, - }, - - After: func(c *cli.Context) error { - if r := recover(); r != nil { - // Generate report in LOTUS_PANIC_REPORT_PATH and re-raise panic - build.GenerateMinerPanicReport(c.String("panic-reports"), c.String(FlagWorkerRepo), c.App.Name) - panic(r) - } - return nil - }, - Commands: local, - } - app.Setup() - app.Metadata["repoType"] = repo.Worker - + app := worker.App() if err := app.Run(os.Args); err != nil { log.Warnf("%+v", err) return } } - -var stopCmd = &cli.Command{ - Name: "stop", - Usage: "Stop a running lotus worker", - Flags: []cli.Flag{}, - Action: func(cctx *cli.Context) error { - api, closer, err := lcli.GetWorkerAPI(cctx) - if err != nil { - return err - } - defer closer() - - ctx := lcli.ReqContext(cctx) - - // Detach any storage associated with this worker - err = api.StorageDetachAll(ctx) - if err != nil { - return err - } - - err = api.Shutdown(ctx) - if err != nil { - return err - } - - return nil - }, -} - -var runCmd = &cli.Command{ - Name: "run", - Usage: "Start lotus worker", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "listen", - Usage: "host address and port the worker api will listen on", - Value: "0.0.0.0:3456", - EnvVars: []string{"LOTUS_WORKER_LISTEN"}, - }, - &cli.StringFlag{ - Name: "address", - Hidden: true, - }, - &cli.BoolFlag{ - Name: "no-local-storage", - Usage: "don't use storageminer repo for sector storage", - EnvVars: []string{"LOTUS_WORKER_NO_LOCAL_STORAGE"}, - }, - &cli.BoolFlag{ - Name: "no-swap", - Usage: "don't use swap", - Value: false, - EnvVars: []string{"LOTUS_WORKER_NO_SWAP"}, - }, - &cli.StringFlag{ - Name: "name", - Usage: "custom worker name", - EnvVars: []string{"LOTUS_WORKER_NAME"}, - DefaultText: "hostname", - }, - &cli.BoolFlag{ - Name: "addpiece", - Usage: "enable addpiece", - Value: true, - EnvVars: []string{"LOTUS_WORKER_ADDPIECE"}, - }, - &cli.BoolFlag{ - Name: "precommit1", - Usage: "enable precommit1", - Value: true, - EnvVars: []string{"LOTUS_WORKER_PRECOMMIT1"}, - }, - &cli.BoolFlag{ - Name: "unseal", - Usage: "enable unsealing", - Value: true, - EnvVars: []string{"LOTUS_WORKER_UNSEAL"}, - }, - &cli.BoolFlag{ - Name: "precommit2", - Usage: "enable precommit2", - Value: true, - EnvVars: []string{"LOTUS_WORKER_PRECOMMIT2"}, - }, - &cli.BoolFlag{ - Name: "commit", - Usage: "enable commit", - Value: true, - EnvVars: []string{"LOTUS_WORKER_COMMIT"}, - }, - &cli.BoolFlag{ - Name: "replica-update", - Usage: "enable replica update", - Value: true, - EnvVars: []string{"LOTUS_WORKER_REPLICA_UPDATE"}, - }, - &cli.BoolFlag{ - Name: "prove-replica-update2", - Usage: "enable prove replica update 2", - Value: true, - EnvVars: []string{"LOTUS_WORKER_PROVE_REPLICA_UPDATE2"}, - }, - &cli.BoolFlag{ - Name: "regen-sector-key", - Usage: "enable regen sector key", - Value: true, - EnvVars: []string{"LOTUS_WORKER_REGEN_SECTOR_KEY"}, - }, - &cli.BoolFlag{ - Name: "sector-download", - Usage: "enable external sector data download", - Value: false, - EnvVars: []string{"LOTUS_WORKER_SECTOR_DOWNLOAD"}, - }, - &cli.BoolFlag{ - Name: "windowpost", - Usage: "enable window post", - Value: false, - EnvVars: []string{"LOTUS_WORKER_WINDOWPOST"}, - }, - &cli.BoolFlag{ - Name: "winningpost", - Usage: "enable winning post", - Value: false, - EnvVars: []string{"LOTUS_WORKER_WINNINGPOST"}, - }, - &cli.BoolFlag{ - Name: "no-default", - Usage: "disable all default compute tasks, use the worker for storage/fetching only", - Value: false, - EnvVars: []string{"LOTUS_WORKER_NO_DEFAULT"}, - }, - &cli.IntFlag{ - Name: "parallel-fetch-limit", - Usage: "maximum fetch operations to run in parallel", - Value: 5, - EnvVars: []string{"LOTUS_WORKER_PARALLEL_FETCH_LIMIT"}, - }, - &cli.IntFlag{ - Name: "post-parallel-reads", - Usage: "maximum number of parallel challenge reads (0 = no limit)", - Value: 32, - EnvVars: []string{"LOTUS_WORKER_POST_PARALLEL_READS"}, - }, - &cli.DurationFlag{ - Name: "post-read-timeout", - Usage: "time limit for reading PoSt challenges (0 = no limit)", - Value: 0, - EnvVars: []string{"LOTUS_WORKER_POST_READ_TIMEOUT"}, - }, - &cli.StringFlag{ - Name: "timeout", - Usage: "used when 'listen' is unspecified. must be a valid duration recognized by golang's time.ParseDuration function", - Value: "30m", - EnvVars: []string{"LOTUS_WORKER_TIMEOUT"}, - }, - &cli.StringFlag{ - Name: "http-server-timeout", - Value: "30s", - }, - &cli.BoolFlag{ - Name: "data-cid", - Usage: "Run the data-cid task. true|false", - Value: true, - DefaultText: "inherits --addpiece", - }, - &cli.StringFlag{ - Name: "external-pc2", - Usage: "command for computing PC2 externally", - }, - }, - Description: `Run lotus-worker. - ---external-pc2 can be used to compute the PreCommit2 inputs externally. -The flag behaves similarly to the related lotus-worker flag, using it in -lotus-bench may be useful for testing if the external PreCommit2 command is -invoked correctly. - -The command will be called with a number of environment variables set: -* EXTSEAL_PC2_SECTOR_NUM: the sector number -* EXTSEAL_PC2_SECTOR_MINER: the miner id -* EXTSEAL_PC2_PROOF_TYPE: the proof type -* EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes -* EXTSEAL_PC2_CACHE: the path to the cache directory -* EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller) -* EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json) - -The command is expected to: -* Create cache sc-02-data-tree-r* files -* Create cache sc-02-data-tree-c* files -* Create cache p_aux / t_aux files -* Transform the sealed file in place - -Example invocation of lotus-bench as external executor: -'./lotus-bench simple precommit2 --sector-size $EXTSEAL_PC2_SECTOR_SIZE $EXTSEAL_PC2_SEALED $EXTSEAL_PC2_CACHE $EXTSEAL_PC2_PC1OUT' -`, - Before: func(cctx *cli.Context) error { - if cctx.IsSet("address") { - log.Warnf("The '--address' flag is deprecated, it has been replaced by '--listen'") - if err := cctx.Set("listen", cctx.String("address")); err != nil { - return err - } - } - - return nil - }, - Action: func(cctx *cli.Context) error { - log.Info("Starting lotus worker") - - if !cctx.Bool("enable-gpu-proving") { - if err := os.Setenv("BELLMAN_NO_GPU", "true"); err != nil { - return xerrors.Errorf("could not set no-gpu env: %+v", err) - } - } - - // ensure tmpdir exists - td := os.TempDir() - if err := os.MkdirAll(td, 0755); err != nil { - return xerrors.Errorf("ensuring temp dir %s exists: %w", td, err) - } - - // Check file descriptor limit - limit, _, err := ulimit.GetLimit() - switch { - case err == ulimit.ErrUnsupported: - log.Errorw("checking file descriptor limit failed", "error", err) - case err != nil: - return xerrors.Errorf("checking fd limit: %w", err) - default: - if limit < buildconstants.MinerFDLimit { - return xerrors.Errorf("soft file descriptor limit (ulimit -n) too low, want %d, current %d", buildconstants.MinerFDLimit, limit) - } - } - - // Check DC-environment variable - sectorSizes := []string{"2KiB", "8MiB", "512MiB", "32GiB", "64GiB"} - resourcesType := reflect.TypeOf(storiface.Resources{}) - - for _, sectorSize := range sectorSizes { - for i := 0; i < resourcesType.NumField(); i++ { - field := resourcesType.Field(i) - envName := field.Tag.Get("envname") - if envName != "" { - // Check if DC_[SectorSize]_[ResourceRestriction] is set - envVar, ok := os.LookupEnv("DC_" + sectorSize + "_" + envName) - if ok { - // If it is set, convert it to DC_[ResourceRestriction] - err := os.Setenv("DC_"+envName, envVar) - if err != nil { - log.Fatalf("Error setting environment variable: %v", err) - } - log.Warnf("Converted DC_%s_%s to DC_%s, because DC is a sector-size independent job", sectorSize, envName, envName) - } - } - } - } - - // Connect to storage-miner - ctx := lcli.ReqContext(cctx) - - // Create a new context with cancel function - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - // Listen for interrupt signals - go func() { - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - <-c - cancel() - }() - - var nodeApi api.StorageMiner - var closer func() - for { - nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp) - if err == nil { - _, err = nodeApi.Version(ctx) - if err == nil { - break - } - } - fmt.Printf("\r\x1b[0KConnecting to miner API... (%s)", err) - select { - case <-ctx.Done(): - return xerrors.New("Interrupted by user") - case <-time.After(time.Second): - } - } - defer closer() - // Register all metric views - if err := view.Register( - metrics.DefaultViews..., - ); err != nil { - log.Fatalf("Cannot register the view: %v", err) - } - - v, err := nodeApi.Version(ctx) - if err != nil { - return err - } - if v.APIVersion != api.MinerAPIVersion0 { - return xerrors.Errorf("lotus-miner API version doesn't match: expected: %s", api.APIVersion{APIVersion: api.MinerAPIVersion0}) - } - log.Infof("Remote version %s", v) - - // Check params - - act, err := nodeApi.ActorAddress(ctx) - if err != nil { - return err - } - ssize, err := nodeApi.ActorSectorSize(ctx, act) - if err != nil { - return err - } - - var taskTypes []sealtasks.TaskType - var workerType string - var needParams bool - - if cctx.Bool("windowpost") { - needParams = true - workerType = sealtasks.WorkerWindowPoSt - taskTypes = append(taskTypes, sealtasks.TTGenerateWindowPoSt) - } - if cctx.Bool("winningpost") { - needParams = true - workerType = sealtasks.WorkerWinningPoSt - taskTypes = append(taskTypes, sealtasks.TTGenerateWinningPoSt) - } - - if workerType == "" { - taskTypes = append(taskTypes, sealtasks.TTFetch, sealtasks.TTCommit1, sealtasks.TTProveReplicaUpdate1, sealtasks.TTFinalize, sealtasks.TTFinalizeUnsealed, sealtasks.TTFinalizeReplicaUpdate) - - if !cctx.Bool("no-default") { - workerType = sealtasks.WorkerSealing - } - } - - ttDataCidDefault := false - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("addpiece")) && cctx.Bool("addpiece") { - taskTypes = append(taskTypes, sealtasks.TTAddPiece) - ttDataCidDefault = true - } - if workerType == sealtasks.WorkerSealing { - if cctx.IsSet("data-cid") { - if cctx.Bool("data-cid") { - taskTypes = append(taskTypes, sealtasks.TTDataCid) - } - } else if ttDataCidDefault { - taskTypes = append(taskTypes, sealtasks.TTDataCid) - } - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("sector-download")) && cctx.Bool("sector-download") { - taskTypes = append(taskTypes, sealtasks.TTDownloadSector) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("precommit1")) && cctx.Bool("precommit1") { - taskTypes = append(taskTypes, sealtasks.TTPreCommit1) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("unseal")) && cctx.Bool("unseal") { - taskTypes = append(taskTypes, sealtasks.TTUnseal) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("precommit2")) && cctx.Bool("precommit2") { - taskTypes = append(taskTypes, sealtasks.TTPreCommit2) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("commit")) && cctx.Bool("commit") { - needParams = true - taskTypes = append(taskTypes, sealtasks.TTCommit2) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("replica-update")) && cctx.Bool("replica-update") { - taskTypes = append(taskTypes, sealtasks.TTReplicaUpdate) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("prove-replica-update2")) && cctx.Bool("prove-replica-update2") { - needParams = true - taskTypes = append(taskTypes, sealtasks.TTProveReplicaUpdate2) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("regen-sector-key")) && cctx.Bool("regen-sector-key") { - taskTypes = append(taskTypes, sealtasks.TTRegenSectorKey) - } - - if cctx.Bool("no-default") && workerType == "" { - workerType = sealtasks.WorkerSealing - } - - if len(taskTypes) == 0 { - return xerrors.Errorf("no task types specified") - } - for _, taskType := range taskTypes { - if taskType.WorkerType() != workerType { - return xerrors.Errorf("expected all task types to be for %s worker, but task %s is for %s worker", workerType, taskType, taskType.WorkerType()) - } - } - - if needParams { - if err := paramfetch.GetParams(ctx, build.ParametersJSON(), build.SrsJSON(), uint64(ssize)); err != nil { - return xerrors.Errorf("get params: %w", err) - } - } - - // Open repo - - repoPath := cctx.String(FlagWorkerRepo) - r, err := repo.NewFS(repoPath) - if err != nil { - return err - } - - ok, err := r.Exists() - if err != nil { - return err - } - if !ok { - if err := r.Init(repo.Worker); err != nil { - return err - } - - lr, err := r.Lock(repo.Worker) - if err != nil { - return err - } - - var localPaths []storiface.LocalPath - - if !cctx.Bool("no-local-storage") { - b, err := json.MarshalIndent(&storiface.LocalStorageMeta{ - ID: storiface.ID(uuid.New().String()), - Weight: 10, - CanSeal: true, - CanStore: false, - }, "", " ") - if err != nil { - return xerrors.Errorf("marshaling storage config: %w", err) - } - - if err := os.WriteFile(filepath.Join(lr.Path(), "sectorstore.json"), b, 0644); err != nil { - return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "sectorstore.json"), err) - } - - localPaths = append(localPaths, storiface.LocalPath{ - Path: lr.Path(), - }) - } - - if err := lr.SetStorage(func(sc *storiface.StorageConfig) { - sc.StoragePaths = append(sc.StoragePaths, localPaths...) - }); err != nil { - return xerrors.Errorf("set storage config: %w", err) - } - - { - // init datastore for r.Exists - _, err := lr.Datastore(context.Background(), "/metadata") - if err != nil { - return err - } - } - if err := lr.Close(); err != nil { - return xerrors.Errorf("close repo: %w", err) - } - } - - lr, err := r.Lock(repo.Worker) - if err != nil { - return err - } - defer func() { - if err := lr.Close(); err != nil { - log.Error("closing repo", err) - } - }() - ds, err := lr.Datastore(context.Background(), "/metadata") - if err != nil { - return err - } - - log.Info("Opening local storage; connecting to master") - const unspecifiedAddress = "0.0.0.0" - - address := cctx.String("listen") - host, port, err := net.SplitHostPort(address) - if err != nil { - return err - } - - if ip := net.ParseIP(host); ip != nil { - if ip.String() == unspecifiedAddress { - timeout, err := time.ParseDuration(cctx.String("timeout")) - if err != nil { - return err - } - rip, err := extractRoutableIP(timeout) - if err != nil { - return err - } - host = rip - } - } - - var newAddress string - - // Check if the IP address is IPv6 - ip := net.ParseIP(host) - if ip.To4() == nil && ip.To16() != nil { - newAddress = "[" + host + "]:" + port - } else { - newAddress = host + ":" + port - } - - localStore, err := paths.NewLocal(ctx, lr, nodeApi, []string{"http://" + newAddress + "/remote"}) - if err != nil { - return err - } - - // Setup remote sector store - sminfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) - if err != nil { - return xerrors.Errorf("could not get api info: %w", err) - } - - remote := paths.NewRemote(localStore, nodeApi, sminfo.AuthHeader(), cctx.Int("parallel-fetch-limit"), - &paths.DefaultPartialFileHandler{}) - - fh := &paths.FetchHandler{Local: localStore, PfHandler: &paths.DefaultPartialFileHandler{}} - remoteHandler := func(w http.ResponseWriter, r *http.Request) { - if !auth.HasPerm(r.Context(), nil, api.PermAdmin) { - w.WriteHeader(401) - _ = json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing admin permission"}) - return - } - - fh.ServeHTTP(w, r) - } - - // Parse ffi executor flags - - var ffiOpts []ffiwrapper.FFIWrapperOpt - - if cctx.IsSet("external-pc2") { - extSeal := ffiwrapper.ExternalSealer{ - PreCommit2: ffiwrapper.MakeExternPrecommit2(cctx.String("external-pc2")), - } - - ffiOpts = append(ffiOpts, ffiwrapper.WithExternalSealCalls(extSeal)) - } - - // Create / expose the worker - - wsts := statestore.New(namespace.Wrap(ds, modules.WorkerCallsPrefix)) - - workerApi := &sealworker.Worker{ - LocalWorker: sealer.NewLocalWorkerWithExecutor( - sealer.FFIExec(ffiOpts...), - sealer.WorkerConfig{ - TaskTypes: taskTypes, - NoSwap: cctx.Bool("no-swap"), - MaxParallelChallengeReads: cctx.Int("post-parallel-reads"), - ChallengeReadTimeout: cctx.Duration("post-read-timeout"), - Name: cctx.String("name"), - }, os.LookupEnv, remote, localStore, nodeApi, nodeApi, wsts), - LocalStore: localStore, - Storage: lr, - } - - log.Info("Setting up control endpoint at " + newAddress) - - timeout, err := time.ParseDuration(cctx.String("http-server-timeout")) - if err != nil { - return xerrors.Errorf("invalid time string %s: %x", cctx.String("http-server-timeout"), err) - } - - srv := &http.Server{ - Handler: sealworker.WorkerHandler(nodeApi.AuthVerify, remoteHandler, workerApi, true), - ReadHeaderTimeout: timeout, - BaseContext: func(listener net.Listener) context.Context { - ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-worker")) - return ctx - }, - } - - go func() { - <-ctx.Done() - log.Warn("Shutting down...") - if err := srv.Shutdown(context.TODO()); err != nil { - log.Errorf("shutting down RPC server failed: %s", err) - } - log.Warn("Graceful shutdown successful") - }() - - nl, err := net.Listen("tcp", newAddress) - if err != nil { - return err - } - - { - a, err := net.ResolveTCPAddr("tcp", newAddress) - if err != nil { - return xerrors.Errorf("parsing address: %w", err) - } - - ma, err := manet.FromNetAddr(a) - if err != nil { - return xerrors.Errorf("creating api multiaddress: %w", err) - } - - if err := lr.SetAPIEndpoint(ma); err != nil { - return xerrors.Errorf("setting api endpoint: %w", err) - } - - ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) - if err != nil { - return xerrors.Errorf("could not get miner API info: %w", err) - } - - // TODO: ideally this would be a token with some permissions dropped - if err := lr.SetAPIToken(ainfo.Token); err != nil { - return xerrors.Errorf("setting api token: %w", err) - } - } - - minerSession, err := nodeApi.Session(ctx) - if err != nil { - return xerrors.Errorf("getting miner session: %w", err) - } - - waitQuietCh := func() chan struct{} { - out := make(chan struct{}) - go func() { - workerApi.LocalWorker.WaitQuiet() - close(out) - }() - return out - } - - go func() { - heartbeats := time.NewTicker(paths.HeartbeatInterval) - defer heartbeats.Stop() - - var redeclareStorage bool - var readyCh chan struct{} - for { - // If we're reconnecting, redeclare storage first - if redeclareStorage { - log.Info("Redeclaring local storage") - - if err := localStore.Redeclare(ctx, nil, false); err != nil { - log.Errorf("Redeclaring local storage failed: %+v", err) - - select { - case <-ctx.Done(): - return // graceful shutdown - case <-heartbeats.C: - } - continue - } - } - - // TODO: we could get rid of this, but that requires tracking resources for restarted tasks correctly - if readyCh == nil { - log.Info("Making sure no local tasks are running") - readyCh = waitQuietCh() - } - - for { - curSession, err := nodeApi.Session(ctx) - if err != nil { - log.Errorf("heartbeat: checking remote session failed: %+v", err) - } else { - if curSession != minerSession { - minerSession = curSession - break - } - } - - select { - case <-readyCh: - if err := nodeApi.WorkerConnect(ctx, "http://"+newAddress+"/rpc/v0"); err != nil { - log.Errorf("Registering worker failed: %+v", err) - cancel() - return - } - - log.Info("Worker registered successfully, waiting for tasks") - - readyCh = nil - case <-heartbeats.C: - case <-ctx.Done(): - return // graceful shutdown - } - } - - log.Errorf("LOTUS-MINER CONNECTION LOST") - - redeclareStorage = true - } - }() - - go func() { - <-workerApi.Done() - // Wait 20s to allow the miner to unregister the worker on next heartbeat - time.Sleep(20 * time.Second) - log.Warn("Shutting down...") - if err := srv.Shutdown(context.TODO()); err != nil { - log.Errorf("shutting down RPC server failed: %s", err) - } - log.Warn("Graceful shutdown successful") - }() - - return srv.Serve(nl) - }, -} - -func extractRoutableIP(timeout time.Duration) (string, error) { - minerMultiAddrKey := "MINER_API_INFO" - deprecatedMinerMultiAddrKey := "STORAGE_API_INFO" - env, ok := os.LookupEnv(minerMultiAddrKey) - if !ok { - _, ok = os.LookupEnv(deprecatedMinerMultiAddrKey) - if ok { - log.Warnf("Using a deprecated env(%s) value, please use env(%s) instead.", deprecatedMinerMultiAddrKey, minerMultiAddrKey) - } - return "", xerrors.New("MINER_API_INFO environment variable required to extract IP") - } - - // Splitting the env to separate the JWT from the multiaddress - splitEnv := strings.SplitN(env, ":", 2) - if len(splitEnv) < 2 { - return "", xerrors.Errorf("invalid MINER_API_INFO format") - } - // Only take the multiaddress part - maddrStr := splitEnv[1] - - maddr, err := multiaddr.NewMultiaddr(maddrStr) - if err != nil { - return "", err - } - - minerIP, _ := maddr.ValueForProtocol(multiaddr.P_IP6) - if minerIP == "" { - minerIP, _ = maddr.ValueForProtocol(multiaddr.P_IP4) - } - minerPort, _ := maddr.ValueForProtocol(multiaddr.P_TCP) - - // Format the address appropriately - addressToDial := net.JoinHostPort(minerIP, minerPort) - - conn, err := net.DialTimeout("tcp", addressToDial, timeout) - if err != nil { - return "", err - } - - defer func() { - if cerr := conn.Close(); cerr != nil { - log.Errorf("Error closing connection: %v", cerr) - } - }() - - localAddr := conn.LocalAddr().(*net.TCPAddr) - return localAddr.IP.String(), nil -} From de3e0a85ff72528f1ebf2c0d16016f6d5c148aad Mon Sep 17 00:00:00 2001 From: Aryan Tikarya Date: Fri, 22 Nov 2024 14:57:10 +0530 Subject: [PATCH 02/15] update the script to generate cli docs directly from command structs --- scripts/docsgen-cli/main.go | 171 +++++++++++++++++++++++++----------- 1 file changed, 121 insertions(+), 50 deletions(-) diff --git a/scripts/docsgen-cli/main.go b/scripts/docsgen-cli/main.go index 95d3c5170d3..a5054634bc2 100644 --- a/scripts/docsgen-cli/main.go +++ b/scripts/docsgen-cli/main.go @@ -3,11 +3,13 @@ package main import ( "fmt" "os" - "os/exec" "path/filepath" "strings" + "github.com/urfave/cli/v2" "golang.org/x/sync/errgroup" + + "github.com/filecoin-project/lotus/cli/worker" ) const ( @@ -46,88 +48,157 @@ func main() { fmt.Println("Generating CLI documentation...") var eg errgroup.Group - for _, cmd := range []string{"lotus", "lotus-miner", "lotus-worker"} { + // Add CLI apps for documentation generation + cliApps := map[string]*cli.App{ + "lotus-worker": worker.App(), + // Add other CLI apps like lotus, lotus-miner as needed + } + + for name, app := range cliApps { eg.Go(func() error { - err := generateMarkdownForCLI(cmd) + err := generateDocsFromApp(name, app) if err != nil { - fmt.Printf(" ❌ %s: %v\n", cmd, err) + fmt.Printf(" ❌ %s: %v\n", name, err) } else { - fmt.Printf(" ✅ %s\n", cmd) + fmt.Printf(" ✅ %s\n", name) } return err }) } + if err := eg.Wait(); err != nil { fmt.Printf("Failed to generate CLI documentation: %v\n", err) os.Exit(1) } + fmt.Println("Documentation generation complete.") } -func generateMarkdownForCLI(cli string) error { - md := filepath.Join(outputDir, fmt.Sprintf("cli-%s.md", cli)) +func generateDocsFromApp(name string, app *cli.App) error { + md := filepath.Join(outputDir, fmt.Sprintf("cli-%s.md", name)) out, err := os.Create(md) if err != nil { return err } defer func() { _ = out.Close() }() - return writeCommandDocs(out, cli, 0) + + // Write top-level documentation + if _, err := out.WriteString(formatAppHeader(app)); err != nil { + return err + } + + return writeDocs(out, app.Commands, 0, name) } -func writeCommandDocs(file *os.File, command string, depth int) error { - // For sanity, fail fast if depth exceeds some arbitrarily large number. In which - // case, chances are there is a bug in this script. - if depth > depthRecursionLimit { - return fmt.Errorf("recursion exceeded limit of %d", depthRecursionLimit) +// formatAppHeader dynamically generates the header for the CLI app. +func formatAppHeader(app *cli.App) string { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("# %s\n\n", app.Name)) + sb.WriteString("```\n") + sb.WriteString(fmt.Sprintf("NAME:\n %s - %s\n\n", app.Name, app.Usage)) + sb.WriteString(fmt.Sprintf("USAGE:\n %s\n\n", generateUsage(app.Name, nil))) // Dynamically fetch usage text + sb.WriteString(fmt.Sprintf("VERSION:\n %s\n\n", app.Version)) + + sb.WriteString("COMMANDS:\n") + for _, cmd := range app.Commands { + sb.WriteString(fmt.Sprintf(" %-10s %s\n", cmd.Name, cmd.Usage)) } - // Get usage from the command. - usage, err := exec.Command("sh", "-c", "./"+command+" -h").Output() - if err != nil { - return fmt.Errorf("failed to run '%s': %v", command, err) + sb.WriteString("\nGLOBAL OPTIONS:\n") + for _, flag := range app.Flags { + sb.WriteString(formatFlag(flag)) } + sb.WriteString("```\n") + return sb.String() +} - // Skip the first new line since the docs do not start with a newline at the very - // top. - if depth != 0 { - if _, err := file.WriteString("\n"); err != nil { - return err - } +// formatFlag formats a cli.Flag into a string for documentation. +func formatFlag(flag cli.Flag) string { + switch f := flag.(type) { + case *cli.StringFlag: + return fmt.Sprintf(" --%-20s %s (default: \"%s\") [%s]\n", f.Name, f.Usage, f.Value, strings.Join(f.EnvVars, ", ")) + case *cli.BoolFlag: + return fmt.Sprintf(" --%-20s %s (default: %t) [%s]\n", f.Name, f.Usage, f.Value, strings.Join(f.EnvVars, ", ")) + case *cli.IntFlag: + return fmt.Sprintf(" --%-20s %s (default: %d) [%s]\n", f.Name, f.Usage, f.Value, strings.Join(f.EnvVars, ", ")) + default: + return fmt.Sprintf(" --%-20s %s\n", flag.Names()[0], flag.String()) } +} - // Write out command header and usage. - header := fmt.Sprintf("%s# %s\n", strings.Repeat("#", depth), command) - if _, err := file.WriteString(header); err != nil { - return err - } else if _, err := file.WriteString("```\n"); err != nil { - return err - } else if _, err := file.Write(usage); err != nil { - return err - } else if _, err := file.WriteString("```\n"); err != nil { - return err +// generateUsage creates the `USAGE` dynamically for root or subcommands. +func generateUsage(parentName string, cmd *cli.Command) string { + if cmd == nil { + // Root command usage + return fmt.Sprintf("%s [global options] command [command options] [arguments...]", parentName) } - // Recurse sub-commands. - commands := false - lines := strings.Split(string(usage), "\n") - for _, line := range lines { - switch line = strings.TrimSpace(line); { - case line == "": - commands = false - case line == "COMMANDS:": - commands = true - case strings.HasPrefix(line, "help, h"): - // Skip usage command. - case commands: - // Find the sub command and trim any potential comma in case of alias. - subCommand := strings.TrimSuffix(strings.Fields(line)[0], ",") - // Skip sections in usage that have no command. - if !strings.Contains(subCommand, ":") { - if err := writeCommandDocs(file, command+" "+subCommand, depth+1); err != nil { + // Subcommand usage + return fmt.Sprintf("%s %s [command options] [arguments...]", parentName, cmd.Name) +} + +func writeDocs(file *os.File, commands []*cli.Command, depth int, rootCommandName string) error { + for _, cmd := range commands { + cmdName := cmd.Name + if rootCommandName != cmdName { + cmdName = rootCommandName + " " + cmdName + } + + header := fmt.Sprintf("\n%s %s\n\n", strings.Repeat("#", depth+2), cmdName) + if _, err := file.WriteString(header); err != nil { + return err + } + + if _, err := file.WriteString("```\n"); err != nil { + return err + } + + // Write command details + if _, err := file.WriteString(fmt.Sprintf("NAME:\n %s - %s\n\n", cmd.Name, cmd.Usage)); err != nil { + return err + } + + if _, err := file.WriteString(fmt.Sprintf("USAGE:\n %s\n\n", generateUsage(rootCommandName, cmd))); err != nil { + return err + } + + if len(cmd.Description) > 0 { + if _, err := file.WriteString(fmt.Sprintf("DESCRIPTION:\n %s\n\n", cmd.Description)); err != nil { + return err + } + } + + // Write options for the command + if len(cmd.Flags) > 0 { + if _, err := file.WriteString("OPTIONS:\n"); err != nil { + return err + } + for _, flag := range cmd.Flags { + if _, err := file.WriteString(formatFlag(flag)); err != nil { return err } } + if _, err := file.WriteString("\n"); err != nil { + return err + } + } + + if len(cmd.Subcommands) > 0 { + if _, err := file.WriteString("```\n"); err != nil { + return err + } + + if err := writeDocs(file, cmd.Subcommands, depth+1, cmdName); err != nil { + return err + } + + continue + } + + if _, err := file.WriteString("```\n"); err != nil { + return err } } + return nil } From ed47ad6ef4d861070a6b04f3b34282bacf39a227 Mon Sep 17 00:00:00 2001 From: Aryan Tikarya Date: Mon, 25 Nov 2024 18:56:29 +0530 Subject: [PATCH 03/15] add args usage --- Makefile | 2 +- scripts/docsgen-cli/main.go | 34 ++++++++++++++++++++++++++++------ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 8197dc2204c..b8575f83f61 100644 --- a/Makefile +++ b/Makefile @@ -349,7 +349,7 @@ snap: lotus lotus-miner lotus-worker # snapcraft upload ./lotus_*.snap # separate from gen because it needs binaries -docsgen-cli: lotus lotus-miner lotus-worker +docsgen-cli: $(GOCC) run ./scripts/docsgen-cli ./lotus config default > documentation/en/default-lotus-config.toml ./lotus-miner config default > documentation/en/default-lotus-miner-config.toml diff --git a/scripts/docsgen-cli/main.go b/scripts/docsgen-cli/main.go index a5054634bc2..165649ce6c7 100644 --- a/scripts/docsgen-cli/main.go +++ b/scripts/docsgen-cli/main.go @@ -96,7 +96,7 @@ func formatAppHeader(app *cli.App) string { sb.WriteString(fmt.Sprintf("# %s\n\n", app.Name)) sb.WriteString("```\n") sb.WriteString(fmt.Sprintf("NAME:\n %s - %s\n\n", app.Name, app.Usage)) - sb.WriteString(fmt.Sprintf("USAGE:\n %s\n\n", generateUsage(app.Name, nil))) // Dynamically fetch usage text + sb.WriteString(fmt.Sprintf("USAGE:\n %s\n\n", generateUsage(app.Name, nil, ""))) // Dynamically fetch usage text sb.WriteString(fmt.Sprintf("VERSION:\n %s\n\n", app.Version)) sb.WriteString("COMMANDS:\n") @@ -127,14 +127,26 @@ func formatFlag(flag cli.Flag) string { } // generateUsage creates the `USAGE` dynamically for root or subcommands. -func generateUsage(parentName string, cmd *cli.Command) string { +func generateUsage(parentName string, cmd *cli.Command, argsUsage string) string { if cmd == nil { // Root command usage return fmt.Sprintf("%s [global options] command [command options] [arguments...]", parentName) } - // Subcommand usage - return fmt.Sprintf("%s %s [command options] [arguments...]", parentName, cmd.Name) + usage := "" + if len(cmd.Subcommands) > 0 { + usage = fmt.Sprintf("%s %s command [command options]", parentName, cmd.Name) + } else { + usage = fmt.Sprintf("%s %s [command options] ", parentName, cmd.Name) + } + + if argsUsage != "" { + usage += " " + argsUsage + } else { + usage += " [arguments...]" + } + + return usage } func writeDocs(file *os.File, commands []*cli.Command, depth int, rootCommandName string) error { @@ -154,11 +166,11 @@ func writeDocs(file *os.File, commands []*cli.Command, depth int, rootCommandNam } // Write command details - if _, err := file.WriteString(fmt.Sprintf("NAME:\n %s - %s\n\n", cmd.Name, cmd.Usage)); err != nil { + if _, err := file.WriteString(fmt.Sprintf("NAME:\n %s - %s\n\n", cmdName, cmd.Usage)); err != nil { return err } - if _, err := file.WriteString(fmt.Sprintf("USAGE:\n %s\n\n", generateUsage(rootCommandName, cmd))); err != nil { + if _, err := file.WriteString(fmt.Sprintf("USAGE:\n %s\n\n", generateUsage(rootCommandName, cmd, cmd.ArgsUsage))); err != nil { return err } @@ -184,6 +196,16 @@ func writeDocs(file *os.File, commands []*cli.Command, depth int, rootCommandNam } if len(cmd.Subcommands) > 0 { + if _, err := file.WriteString("COMMANDS:\n"); err != nil { + return err + } + + for _, subcmd := range cmd.Subcommands { + if _, err := file.WriteString(fmt.Sprintf(" %-10s %s\n", subcmd.Name, subcmd.Usage)); err != nil { + return err + } + } + if _, err := file.WriteString("```\n"); err != nil { return err } From e3c452e6b114882896bd5b0917eee14a4d76b305 Mon Sep 17 00:00:00 2001 From: Aryan Tikarya Date: Mon, 25 Nov 2024 19:40:13 +0530 Subject: [PATCH 04/15] refactor: lotus commands --- {cmd => cli}/lotus/backup.go | 2 +- {cmd => cli}/lotus/config.go | 2 +- {cmd => cli}/lotus/daemon.go | 2 +- {cmd => cli}/lotus/daemon_nodaemon.go | 2 +- {cmd => cli}/lotus/debug_advance.go | 2 +- cli/lotus/lotus.go | 123 ++++++++++++++++++++++++++ cmd/lotus/main.go | 117 +----------------------- scripts/docsgen-cli/main.go | 9 +- 8 files changed, 135 insertions(+), 124 deletions(-) rename {cmd => cli}/lotus/backup.go (99%) rename {cmd => cli}/lotus/config.go (99%) rename {cmd => cli}/lotus/daemon.go (99%) rename {cmd => cli}/lotus/daemon_nodaemon.go (96%) rename {cmd => cli}/lotus/debug_advance.go (99%) create mode 100644 cli/lotus/lotus.go diff --git a/cmd/lotus/backup.go b/cli/lotus/backup.go similarity index 99% rename from cmd/lotus/backup.go rename to cli/lotus/backup.go index 0fbdf29622c..ed664a370aa 100644 --- a/cmd/lotus/backup.go +++ b/cli/lotus/backup.go @@ -1,4 +1,4 @@ -package main +package lotus import ( "os" diff --git a/cmd/lotus/config.go b/cli/lotus/config.go similarity index 99% rename from cmd/lotus/config.go rename to cli/lotus/config.go index 4b323bfb227..9528cc05363 100644 --- a/cmd/lotus/config.go +++ b/cli/lotus/config.go @@ -1,4 +1,4 @@ -package main +package lotus import ( "fmt" diff --git a/cmd/lotus/daemon.go b/cli/lotus/daemon.go similarity index 99% rename from cmd/lotus/daemon.go rename to cli/lotus/daemon.go index b7fbd63e695..bd364340309 100644 --- a/cmd/lotus/daemon.go +++ b/cli/lotus/daemon.go @@ -1,7 +1,7 @@ //go:build !nodaemon // +build !nodaemon -package main +package lotus import ( "bufio" diff --git a/cmd/lotus/daemon_nodaemon.go b/cli/lotus/daemon_nodaemon.go similarity index 96% rename from cmd/lotus/daemon_nodaemon.go rename to cli/lotus/daemon_nodaemon.go index 7cf12dac41a..be3f8721e58 100644 --- a/cmd/lotus/daemon_nodaemon.go +++ b/cli/lotus/daemon_nodaemon.go @@ -1,7 +1,7 @@ //go:build nodaemon // +build nodaemon -package main +package lotus import ( "errors" diff --git a/cmd/lotus/debug_advance.go b/cli/lotus/debug_advance.go similarity index 99% rename from cmd/lotus/debug_advance.go rename to cli/lotus/debug_advance.go index 422f14e5a2e..915af49d668 100644 --- a/cmd/lotus/debug_advance.go +++ b/cli/lotus/debug_advance.go @@ -1,7 +1,7 @@ //go:build debug // +build debug -package main +package lotus import ( "encoding/binary" diff --git a/cli/lotus/lotus.go b/cli/lotus/lotus.go new file mode 100644 index 00000000000..908b0c57c71 --- /dev/null +++ b/cli/lotus/lotus.go @@ -0,0 +1,123 @@ +package lotus + +import ( + "context" + "os" + + "github.com/fatih/color" + cliutil "github.com/filecoin-project/lotus/cli/util" + logging "github.com/ipfs/go-log/v2" + "github.com/mattn/go-isatty" + "github.com/urfave/cli/v2" + "go.opencensus.io/trace" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/cli/clicommands" + "github.com/filecoin-project/lotus/lib/lotuslog" + "github.com/filecoin-project/lotus/lib/tracing" + "github.com/filecoin-project/lotus/node/repo" +) + +var log = logging.Logger("lotus") + +var AdvanceBlockCmd *cli.Command + +func App() *cli.App { + api.RunningNodeType = api.NodeFull + + lotuslog.SetupLogLevels() + + local := []*cli.Command{ + DaemonCmd, + backupCmd, + configCmd, + } + if AdvanceBlockCmd != nil { + local = append(local, AdvanceBlockCmd) + } + + jaeger := tracing.SetupJaegerTracing("lotus") + defer func() { + if jaeger != nil { + _ = jaeger.ForceFlush(context.Background()) + } + }() + + for _, cmd := range local { + cmd := cmd + originBefore := cmd.Before + cmd.Before = func(cctx *cli.Context) error { + if jaeger != nil { + _ = jaeger.Shutdown(cctx.Context) + } + jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name) + + if cctx.IsSet("color") { + color.NoColor = !cctx.Bool("color") + } + + if originBefore != nil { + return originBefore(cctx) + } + return nil + } + } + ctx, span := trace.StartSpan(context.Background(), "/cli") + defer span.End() + + interactiveDef := isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) + + app := &cli.App{ + Name: "lotus", + Usage: "Filecoin decentralized storage network client", + Version: string(build.NodeUserVersion()), + EnableBashCompletion: true, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "panic-reports", + EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, + Hidden: true, + Value: "~/.lotus", // should follow --repo default + }, + &cli.BoolFlag{ + // examined in the Before above + Name: "color", + Usage: "use color in display output", + DefaultText: "depends on output being a TTY", + }, + &cli.StringFlag{ + Name: "repo", + EnvVars: []string{"LOTUS_PATH"}, + Hidden: true, + Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME + }, + &cli.BoolFlag{ + Name: "interactive", + Usage: "setting to false will disable interactive functionality of commands", + Value: interactiveDef, + }, + &cli.BoolFlag{ + Name: "force-send", + Usage: "if true, will ignore pre-send checks", + }, + cliutil.FlagVeryVerbose, + }, + After: func(c *cli.Context) error { + if r := recover(); r != nil { + // Generate report in LOTUS_PATH and re-raise panic + build.GenerateNodePanicReport(c.String("panic-reports"), c.String("repo"), c.App.Name) + panic(r) + } + return nil + }, + + Commands: append(local, clicommands.Commands...), + } + + app.Setup() + app.Metadata["traceContext"] = ctx + app.Metadata["repoType"] = repo.FullNode + + return app +} diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index 6b29f6cf83a..4778689d20e 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -1,124 +1,11 @@ package main import ( - "context" - "os" - - "github.com/fatih/color" - logging "github.com/ipfs/go-log/v2" - "github.com/mattn/go-isatty" - "github.com/urfave/cli/v2" - "go.opencensus.io/trace" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/cli/clicommands" - cliutil "github.com/filecoin-project/lotus/cli/util" - "github.com/filecoin-project/lotus/lib/lotuslog" - "github.com/filecoin-project/lotus/lib/tracing" - "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/cli/lotus" ) -var log = logging.Logger("main") - -var AdvanceBlockCmd *cli.Command - func main() { - api.RunningNodeType = api.NodeFull - - lotuslog.SetupLogLevels() - - local := []*cli.Command{ - DaemonCmd, - backupCmd, - configCmd, - } - if AdvanceBlockCmd != nil { - local = append(local, AdvanceBlockCmd) - } - - jaeger := tracing.SetupJaegerTracing("lotus") - defer func() { - if jaeger != nil { - _ = jaeger.ForceFlush(context.Background()) - } - }() - - for _, cmd := range local { - cmd := cmd - originBefore := cmd.Before - cmd.Before = func(cctx *cli.Context) error { - if jaeger != nil { - _ = jaeger.Shutdown(cctx.Context) - } - jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name) - - if cctx.IsSet("color") { - color.NoColor = !cctx.Bool("color") - } - - if originBefore != nil { - return originBefore(cctx) - } - return nil - } - } - ctx, span := trace.StartSpan(context.Background(), "/cli") - defer span.End() - - interactiveDef := isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) - - app := &cli.App{ - Name: "lotus", - Usage: "Filecoin decentralized storage network client", - Version: string(build.NodeUserVersion()), - EnableBashCompletion: true, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "panic-reports", - EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, - Hidden: true, - Value: "~/.lotus", // should follow --repo default - }, - &cli.BoolFlag{ - // examined in the Before above - Name: "color", - Usage: "use color in display output", - DefaultText: "depends on output being a TTY", - }, - &cli.StringFlag{ - Name: "repo", - EnvVars: []string{"LOTUS_PATH"}, - Hidden: true, - Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME - }, - &cli.BoolFlag{ - Name: "interactive", - Usage: "setting to false will disable interactive functionality of commands", - Value: interactiveDef, - }, - &cli.BoolFlag{ - Name: "force-send", - Usage: "if true, will ignore pre-send checks", - }, - cliutil.FlagVeryVerbose, - }, - After: func(c *cli.Context) error { - if r := recover(); r != nil { - // Generate report in LOTUS_PATH and re-raise panic - build.GenerateNodePanicReport(c.String("panic-reports"), c.String("repo"), c.App.Name) - panic(r) - } - return nil - }, - - Commands: append(local, clicommands.Commands...), - } - - app.Setup() - app.Metadata["traceContext"] = ctx - app.Metadata["repoType"] = repo.FullNode - + app := lotus.App() lcli.RunApp(app) } diff --git a/scripts/docsgen-cli/main.go b/scripts/docsgen-cli/main.go index 165649ce6c7..f150d5a876a 100644 --- a/scripts/docsgen-cli/main.go +++ b/scripts/docsgen-cli/main.go @@ -9,6 +9,7 @@ import ( "github.com/urfave/cli/v2" "golang.org/x/sync/errgroup" + "github.com/filecoin-project/lotus/cli/lotus" "github.com/filecoin-project/lotus/cli/worker" ) @@ -50,8 +51,8 @@ func main() { var eg errgroup.Group // Add CLI apps for documentation generation cliApps := map[string]*cli.App{ + "lotus": lotus.App(), "lotus-worker": worker.App(), - // Add other CLI apps like lotus, lotus-miner as needed } for name, app := range cliApps { @@ -135,15 +136,15 @@ func generateUsage(parentName string, cmd *cli.Command, argsUsage string) string usage := "" if len(cmd.Subcommands) > 0 { - usage = fmt.Sprintf("%s %s command [command options]", parentName, cmd.Name) + usage = fmt.Sprintf("%s %s command [command options] ", parentName, cmd.Name) } else { usage = fmt.Sprintf("%s %s [command options] ", parentName, cmd.Name) } if argsUsage != "" { - usage += " " + argsUsage + usage += argsUsage } else { - usage += " [arguments...]" + usage += "[arguments...]" } return usage From 8233c8e019192d9b0c85a0c65b9c6de5458a5ec9 Mon Sep 17 00:00:00 2001 From: Aryan Tikarya Date: Mon, 25 Nov 2024 21:23:58 +0530 Subject: [PATCH 05/15] refactor: lotus-miner commands --- {cmd/lotus-miner => cli/miner}/actor.go | 2 +- {cmd/lotus-miner => cli/miner}/actor_test.go | 4 +- .../lotus-miner => cli/miner}/allinfo_test.go | 2 +- {cmd/lotus-miner => cli/miner}/backup.go | 2 +- {cmd/lotus-miner => cli/miner}/config.go | 2 +- {cmd/lotus-miner => cli/miner}/info.go | 2 +- {cmd/lotus-miner => cli/miner}/info_all.go | 2 +- {cmd/lotus-miner => cli/miner}/init.go | 2 +- .../lotus-miner => cli/miner}/init_restore.go | 2 +- cli/miner/miner.go | 167 ++++++++++++++++++ {cmd/lotus-miner => cli/miner}/proving.go | 4 +- {cmd/lotus-miner => cli/miner}/run.go | 2 +- {cmd/lotus-miner => cli/miner}/sealing.go | 2 +- {cmd/lotus-miner => cli/miner}/sectors.go | 2 +- {cmd/lotus-miner => cli/miner}/stop.go | 2 +- {cmd/lotus-miner => cli/miner}/storage.go | 2 +- cmd/lotus-miner/main.go | 161 +---------------- scripts/docsgen-cli/main.go | 2 + 18 files changed, 188 insertions(+), 176 deletions(-) rename {cmd/lotus-miner => cli/miner}/actor.go (99%) rename {cmd/lotus-miner => cli/miner}/actor_test.go (98%) rename {cmd/lotus-miner => cli/miner}/allinfo_test.go (98%) rename {cmd/lotus-miner => cli/miner}/backup.go (96%) rename {cmd/lotus-miner => cli/miner}/config.go (99%) rename {cmd/lotus-miner => cli/miner}/info.go (99%) rename {cmd/lotus-miner => cli/miner}/info_all.go (99%) rename {cmd/lotus-miner => cli/miner}/init.go (99%) rename {cmd/lotus-miner => cli/miner}/init_restore.go (99%) create mode 100644 cli/miner/miner.go rename {cmd/lotus-miner => cli/miner}/proving.go (99%) rename {cmd/lotus-miner => cli/miner}/run.go (99%) rename {cmd/lotus-miner => cli/miner}/sealing.go (99%) rename {cmd/lotus-miner => cli/miner}/sectors.go (99%) rename {cmd/lotus-miner => cli/miner}/stop.go (96%) rename {cmd/lotus-miner => cli/miner}/storage.go (99%) diff --git a/cmd/lotus-miner/actor.go b/cli/miner/actor.go similarity index 99% rename from cmd/lotus-miner/actor.go rename to cli/miner/actor.go index 1330d883924..89eae061304 100644 --- a/cmd/lotus-miner/actor.go +++ b/cli/miner/actor.go @@ -1,4 +1,4 @@ -package main +package miner import ( "fmt" diff --git a/cmd/lotus-miner/actor_test.go b/cli/miner/actor_test.go similarity index 98% rename from cmd/lotus-miner/actor_test.go rename to cli/miner/actor_test.go index 5f9e923e680..172f29d32b3 100644 --- a/cmd/lotus-miner/actor_test.go +++ b/cli/miner/actor_test.go @@ -1,5 +1,5 @@ // stm: #unit -package main +package miner import ( "bytes" @@ -25,7 +25,7 @@ import ( ) func TestWorkerKeyChange(t *testing.T) { - //stm: @OTHER_WORKER_KEY_CHANGE_001 + // stm: @OTHER_WORKER_KEY_CHANGE_001 if testing.Short() { t.Skip("skipping test in short mode") } diff --git a/cmd/lotus-miner/allinfo_test.go b/cli/miner/allinfo_test.go similarity index 98% rename from cmd/lotus-miner/allinfo_test.go rename to cli/miner/allinfo_test.go index a0735fc9771..8943b7d93a5 100644 --- a/cmd/lotus-miner/allinfo_test.go +++ b/cli/miner/allinfo_test.go @@ -1,5 +1,5 @@ // stm: #integration -package main +package miner import ( "flag" diff --git a/cmd/lotus-miner/backup.go b/cli/miner/backup.go similarity index 96% rename from cmd/lotus-miner/backup.go rename to cli/miner/backup.go index cf8c9f9125c..b7c8cc9de50 100644 --- a/cmd/lotus-miner/backup.go +++ b/cli/miner/backup.go @@ -1,4 +1,4 @@ -package main +package miner import ( "github.com/urfave/cli/v2" diff --git a/cmd/lotus-miner/config.go b/cli/miner/config.go similarity index 99% rename from cmd/lotus-miner/config.go rename to cli/miner/config.go index b7af1b2e51e..16e3061ac0c 100644 --- a/cmd/lotus-miner/config.go +++ b/cli/miner/config.go @@ -1,4 +1,4 @@ -package main +package miner import ( "fmt" diff --git a/cmd/lotus-miner/info.go b/cli/miner/info.go similarity index 99% rename from cmd/lotus-miner/info.go rename to cli/miner/info.go index 9548a4a771f..992c1100ef3 100644 --- a/cmd/lotus-miner/info.go +++ b/cli/miner/info.go @@ -1,4 +1,4 @@ -package main +package miner import ( "context" diff --git a/cmd/lotus-miner/info_all.go b/cli/miner/info_all.go similarity index 99% rename from cmd/lotus-miner/info_all.go rename to cli/miner/info_all.go index 253d2befc67..250cc5ad00f 100644 --- a/cmd/lotus-miner/info_all.go +++ b/cli/miner/info_all.go @@ -1,4 +1,4 @@ -package main +package miner import ( "flag" diff --git a/cmd/lotus-miner/init.go b/cli/miner/init.go similarity index 99% rename from cmd/lotus-miner/init.go rename to cli/miner/init.go index 6d6fe7362a5..4f84b9c1c84 100644 --- a/cmd/lotus-miner/init.go +++ b/cli/miner/init.go @@ -1,4 +1,4 @@ -package main +package miner import ( "bytes" diff --git a/cmd/lotus-miner/init_restore.go b/cli/miner/init_restore.go similarity index 99% rename from cmd/lotus-miner/init_restore.go rename to cli/miner/init_restore.go index 05721f6cf56..e8e8970a1e8 100644 --- a/cmd/lotus-miner/init_restore.go +++ b/cli/miner/init_restore.go @@ -1,4 +1,4 @@ -package main +package miner import ( "context" diff --git a/cli/miner/miner.go b/cli/miner/miner.go new file mode 100644 index 00000000000..e3185c507df --- /dev/null +++ b/cli/miner/miner.go @@ -0,0 +1,167 @@ +package miner + +import ( + "context" + "fmt" + + "github.com/fatih/color" + "github.com/filecoin-project/go-address" + logging "github.com/ipfs/go-log/v2" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + lcli "github.com/filecoin-project/lotus/cli" + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/lib/lotuslog" + "github.com/filecoin-project/lotus/lib/tracing" + "github.com/filecoin-project/lotus/node/repo" +) + +var log = logging.Logger("lotus-miner") + +const ( + FlagMinerRepo = "miner-repo" +) + +// TODO remove after deprecation period +const FlagMinerRepoDeprecation = "storagerepo" + +func App() *cli.App { + api.RunningNodeType = api.NodeMiner + + lotuslog.SetupLogLevels() + + local := []*cli.Command{ + initCmd, + runCmd, + stopCmd, + configCmd, + backupCmd, + lcli.WithCategory("chain", actorCmd), + lcli.WithCategory("chain", infoCmd), + lcli.WithCategory("storage", sectorsCmd), + lcli.WithCategory("storage", provingCmd), + lcli.WithCategory("storage", storageCmd), + lcli.WithCategory("storage", sealingCmd), + } + + jaeger := tracing.SetupJaegerTracing("lotus") + defer func() { + if jaeger != nil { + _ = jaeger.ForceFlush(context.Background()) + } + }() + + for _, cmd := range local { + cmd := cmd + originBefore := cmd.Before + cmd.Before = func(cctx *cli.Context) error { + if jaeger != nil { + _ = jaeger.Shutdown(cctx.Context) + } + jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name) + + if cctx.IsSet("color") { + color.NoColor = !cctx.Bool("color") + } + + if originBefore != nil { + return originBefore(cctx) + } + + return nil + } + } + + app := &cli.App{ + Name: "lotus-miner", + Usage: "Filecoin decentralized storage network miner", + Version: string(build.MinerUserVersion()), + EnableBashCompletion: true, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "actor", + Value: "", + Usage: "specify other actor to query / manipulate", + Aliases: []string{"a"}, + }, + &cli.BoolFlag{ + // examined in the Before above + Name: "color", + Usage: "use color in display output", + DefaultText: "depends on output being a TTY", + }, + &cli.StringFlag{ + Name: "panic-reports", + EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, + Hidden: true, + Value: "~/.lotusminer", // should follow --repo default + }, + &cli.StringFlag{ + Name: "repo", + EnvVars: []string{"LOTUS_PATH"}, + Hidden: true, + Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME + }, + &cli.StringFlag{ + Name: FlagMinerRepo, + Aliases: []string{FlagMinerRepoDeprecation}, + EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"}, + Value: "~/.lotusminer", // TODO: Consider XDG_DATA_HOME + Usage: fmt.Sprintf("Specify miner repo path. flag(%s) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON", FlagMinerRepoDeprecation), + }, + cliutil.FlagVeryVerbose, + }, + Commands: append(local, lcli.CommonCommands...), + After: func(c *cli.Context) error { + if r := recover(); r != nil { + // Generate report in LOTUS_PATH and re-raise panic + build.GenerateMinerPanicReport(c.String("panic-reports"), c.String(FlagMinerRepo), c.App.Name) + panic(r) + } + return nil + }, + } + app.Setup() + app.Metadata["repoType"] = repo.StorageMiner + return app +} + +func getActorAddress(ctx context.Context, cctx *cli.Context) (maddr address.Address, err error) { + if cctx.IsSet("actor") { + maddr, err = address.NewFromString(cctx.String("actor")) + if err != nil { + return maddr, err + } + return + } + + minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return address.Undef, err + } + defer closer() + + maddr, err = minerApi.ActorAddress(ctx) + if err != nil { + return maddr, xerrors.Errorf("getting actor address: %w", err) + } + + return maddr, nil +} + +func LMActorOrEnvGetter(cctx *cli.Context) (address.Address, error) { + return getActorAddress(cctx.Context, cctx) +} + +func LMActorGetter(cctx *cli.Context) (address.Address, error) { + minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return address.Undef, err + } + defer closer() + + return minerApi.ActorAddress(cctx.Context) +} diff --git a/cmd/lotus-miner/proving.go b/cli/miner/proving.go similarity index 99% rename from cmd/lotus-miner/proving.go rename to cli/miner/proving.go index 79ad623d1ca..6b87e13ea17 100644 --- a/cmd/lotus-miner/proving.go +++ b/cli/miner/proving.go @@ -1,4 +1,4 @@ -package main +package miner import ( "encoding/json" @@ -244,7 +244,7 @@ It will not send any messages to the chain.`, return err } - //convert sector information into easily readable information + // convert sector information into easily readable information type PoStPartition struct { Index uint64 Skipped []uint64 diff --git a/cmd/lotus-miner/run.go b/cli/miner/run.go similarity index 99% rename from cmd/lotus-miner/run.go rename to cli/miner/run.go index e09968165b9..f3edca08c41 100644 --- a/cmd/lotus-miner/run.go +++ b/cli/miner/run.go @@ -1,4 +1,4 @@ -package main +package miner import ( "fmt" diff --git a/cmd/lotus-miner/sealing.go b/cli/miner/sealing.go similarity index 99% rename from cmd/lotus-miner/sealing.go rename to cli/miner/sealing.go index ed2b2d2948a..0b807b31348 100644 --- a/cmd/lotus-miner/sealing.go +++ b/cli/miner/sealing.go @@ -1,4 +1,4 @@ -package main +package miner import ( "encoding/hex" diff --git a/cmd/lotus-miner/sectors.go b/cli/miner/sectors.go similarity index 99% rename from cmd/lotus-miner/sectors.go rename to cli/miner/sectors.go index fa23e5f268f..c9eb8e777e5 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cli/miner/sectors.go @@ -1,4 +1,4 @@ -package main +package miner import ( "bufio" diff --git a/cmd/lotus-miner/stop.go b/cli/miner/stop.go similarity index 96% rename from cmd/lotus-miner/stop.go rename to cli/miner/stop.go index 186a156169e..f5bfeaa4889 100644 --- a/cmd/lotus-miner/stop.go +++ b/cli/miner/stop.go @@ -1,4 +1,4 @@ -package main +package miner import ( _ "net/http/pprof" diff --git a/cmd/lotus-miner/storage.go b/cli/miner/storage.go similarity index 99% rename from cmd/lotus-miner/storage.go rename to cli/miner/storage.go index b39fe2bf750..2fcf60ff3db 100644 --- a/cmd/lotus-miner/storage.go +++ b/cli/miner/storage.go @@ -1,4 +1,4 @@ -package main +package miner import ( "context" diff --git a/cmd/lotus-miner/main.go b/cmd/lotus-miner/main.go index 76b8c0deb05..55ab1e6a132 100644 --- a/cmd/lotus-miner/main.go +++ b/cmd/lotus-miner/main.go @@ -1,168 +1,11 @@ package main import ( - "context" - "fmt" - - "github.com/fatih/color" - logging "github.com/ipfs/go-log/v2" - "github.com/urfave/cli/v2" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" - cliutil "github.com/filecoin-project/lotus/cli/util" - "github.com/filecoin-project/lotus/lib/lotuslog" - "github.com/filecoin-project/lotus/lib/tracing" - "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/cli/miner" ) -var log = logging.Logger("main") - -const ( - FlagMinerRepo = "miner-repo" -) - -// TODO remove after deprecation period -const FlagMinerRepoDeprecation = "storagerepo" - func main() { - api.RunningNodeType = api.NodeMiner - - lotuslog.SetupLogLevels() - - local := []*cli.Command{ - initCmd, - runCmd, - stopCmd, - configCmd, - backupCmd, - lcli.WithCategory("chain", actorCmd), - lcli.WithCategory("chain", infoCmd), - lcli.WithCategory("storage", sectorsCmd), - lcli.WithCategory("storage", provingCmd), - lcli.WithCategory("storage", storageCmd), - lcli.WithCategory("storage", sealingCmd), - } - - jaeger := tracing.SetupJaegerTracing("lotus") - defer func() { - if jaeger != nil { - _ = jaeger.ForceFlush(context.Background()) - } - }() - - for _, cmd := range local { - cmd := cmd - originBefore := cmd.Before - cmd.Before = func(cctx *cli.Context) error { - if jaeger != nil { - _ = jaeger.Shutdown(cctx.Context) - } - jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name) - - if cctx.IsSet("color") { - color.NoColor = !cctx.Bool("color") - } - - if originBefore != nil { - return originBefore(cctx) - } - - return nil - } - } - - app := &cli.App{ - Name: "lotus-miner", - Usage: "Filecoin decentralized storage network miner", - Version: string(build.MinerUserVersion()), - EnableBashCompletion: true, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "actor", - Value: "", - Usage: "specify other actor to query / manipulate", - Aliases: []string{"a"}, - }, - &cli.BoolFlag{ - // examined in the Before above - Name: "color", - Usage: "use color in display output", - DefaultText: "depends on output being a TTY", - }, - &cli.StringFlag{ - Name: "panic-reports", - EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, - Hidden: true, - Value: "~/.lotusminer", // should follow --repo default - }, - &cli.StringFlag{ - Name: "repo", - EnvVars: []string{"LOTUS_PATH"}, - Hidden: true, - Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME - }, - &cli.StringFlag{ - Name: FlagMinerRepo, - Aliases: []string{FlagMinerRepoDeprecation}, - EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"}, - Value: "~/.lotusminer", // TODO: Consider XDG_DATA_HOME - Usage: fmt.Sprintf("Specify miner repo path. flag(%s) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON", FlagMinerRepoDeprecation), - }, - cliutil.FlagVeryVerbose, - }, - Commands: append(local, lcli.CommonCommands...), - After: func(c *cli.Context) error { - if r := recover(); r != nil { - // Generate report in LOTUS_PATH and re-raise panic - build.GenerateMinerPanicReport(c.String("panic-reports"), c.String(FlagMinerRepo), c.App.Name) - panic(r) - } - return nil - }, - } - app.Setup() - app.Metadata["repoType"] = repo.StorageMiner + app := miner.App() lcli.RunApp(app) } - -func getActorAddress(ctx context.Context, cctx *cli.Context) (maddr address.Address, err error) { - if cctx.IsSet("actor") { - maddr, err = address.NewFromString(cctx.String("actor")) - if err != nil { - return maddr, err - } - return - } - - minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return address.Undef, err - } - defer closer() - - maddr, err = minerApi.ActorAddress(ctx) - if err != nil { - return maddr, xerrors.Errorf("getting actor address: %w", err) - } - - return maddr, nil -} - -func LMActorOrEnvGetter(cctx *cli.Context) (address.Address, error) { - return getActorAddress(cctx.Context, cctx) -} - -func LMActorGetter(cctx *cli.Context) (address.Address, error) { - minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return address.Undef, err - } - defer closer() - - return minerApi.ActorAddress(cctx.Context) -} diff --git a/scripts/docsgen-cli/main.go b/scripts/docsgen-cli/main.go index f150d5a876a..3874391a98f 100644 --- a/scripts/docsgen-cli/main.go +++ b/scripts/docsgen-cli/main.go @@ -10,6 +10,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/filecoin-project/lotus/cli/lotus" + "github.com/filecoin-project/lotus/cli/miner" "github.com/filecoin-project/lotus/cli/worker" ) @@ -53,6 +54,7 @@ func main() { cliApps := map[string]*cli.App{ "lotus": lotus.App(), "lotus-worker": worker.App(), + "lotus-miner": miner.App(), } for name, app := range cliApps { From a8b4a757e796a93f02240f7ceb86546678616891 Mon Sep 17 00:00:00 2001 From: Aryan Tikarya Date: Tue, 26 Nov 2024 20:42:46 +0530 Subject: [PATCH 06/15] refactor: cli docs generation --- Makefile | 3 - cli/lotus/lotus.go | 2 +- cli/miner/miner.go | 3 +- cli/worker/worker.go | 1 + cmd/lotus-worker/main.go | 3 +- documentation/en/cli-lotus-miner.md | 1159 ++++++------ documentation/en/cli-lotus-worker.md | 236 ++- documentation/en/cli-lotus.md | 1897 +++++++++++--------- documentation/en/default-lotus-config.toml | 380 ---- scripts/docsgen-cli/cmd_formatter.go | 72 + scripts/docsgen-cli/doc_generator.go | 105 ++ scripts/docsgen-cli/flag_formatter.go | 68 + scripts/docsgen-cli/main.go | 174 +- scripts/docsgen-cli/utils.go | 44 + 14 files changed, 2070 insertions(+), 2077 deletions(-) create mode 100644 scripts/docsgen-cli/cmd_formatter.go create mode 100644 scripts/docsgen-cli/doc_generator.go create mode 100644 scripts/docsgen-cli/flag_formatter.go create mode 100644 scripts/docsgen-cli/utils.go diff --git a/Makefile b/Makefile index b8575f83f61..4c622d12855 100644 --- a/Makefile +++ b/Makefile @@ -348,11 +348,8 @@ snap: lotus lotus-miner lotus-worker snapcraft # snapcraft upload ./lotus_*.snap -# separate from gen because it needs binaries docsgen-cli: $(GOCC) run ./scripts/docsgen-cli - ./lotus config default > documentation/en/default-lotus-config.toml - ./lotus-miner config default > documentation/en/default-lotus-miner-config.toml .PHONY: docsgen-cli print-%: diff --git a/cli/lotus/lotus.go b/cli/lotus/lotus.go index 908b0c57c71..1bdaf6fd217 100644 --- a/cli/lotus/lotus.go +++ b/cli/lotus/lotus.go @@ -5,7 +5,6 @@ import ( "os" "github.com/fatih/color" - cliutil "github.com/filecoin-project/lotus/cli/util" logging "github.com/ipfs/go-log/v2" "github.com/mattn/go-isatty" "github.com/urfave/cli/v2" @@ -14,6 +13,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/cli/clicommands" + cliutil "github.com/filecoin-project/lotus/cli/util" "github.com/filecoin-project/lotus/lib/lotuslog" "github.com/filecoin-project/lotus/lib/tracing" "github.com/filecoin-project/lotus/node/repo" diff --git a/cli/miner/miner.go b/cli/miner/miner.go index e3185c507df..3d432ec99b3 100644 --- a/cli/miner/miner.go +++ b/cli/miner/miner.go @@ -5,11 +5,12 @@ import ( "fmt" "github.com/fatih/color" - "github.com/filecoin-project/go-address" logging "github.com/ipfs/go-log/v2" "github.com/urfave/cli/v2" "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" diff --git a/cli/worker/worker.go b/cli/worker/worker.go index 3255d55bd17..64580821284 100644 --- a/cli/worker/worker.go +++ b/cli/worker/worker.go @@ -26,6 +26,7 @@ import ( "github.com/filecoin-project/go-jsonrpc/auth" "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-statestore" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/build/buildconstants" diff --git a/cmd/lotus-worker/main.go b/cmd/lotus-worker/main.go index b67da77f21c..7b120499a7d 100644 --- a/cmd/lotus-worker/main.go +++ b/cmd/lotus-worker/main.go @@ -3,10 +3,11 @@ package main import ( "os" + logging "github.com/ipfs/go-log/v2" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/cli/worker" "github.com/filecoin-project/lotus/lib/lotuslog" - logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("main") diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index 7676a4e88e7..ef2da72e530 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -1,4 +1,5 @@ # lotus-miner + ``` NAME: lotus-miner - Filecoin decentralized storage network miner @@ -10,36 +11,38 @@ VERSION: 1.31.1-dev COMMANDS: - init Initialize a lotus miner repo - run Start a lotus miner process - stop Stop a running lotus miner - config Manage node config - backup Create node metadata backup - auth Manage RPC permissions - log Manage logging - wait-api Wait for lotus api to come online - fetch-params Fetch proving parameters - version Print version - help, h Shows a list of commands or help for one command - CHAIN: - actor manipulate the miner actor - info Print miner info - STORAGE: - sectors interact with sector store - proving View proving information - storage manage sector storage - sealing interact with sealing pipeline + init Initialize a lotus miner repo + run Start a lotus miner process + stop Stop a running lotus miner + config Manage node config + backup Create node metadata backup + actor manipulate the miner actor + info Print miner info + sectors interact with sector store + proving View proving information + storage manage sector storage + sealing interact with sealing pipeline + auth Manage RPC permissions + log Manage logging + wait-api Wait for lotus api to come online + fetch-params Fetch proving parameters + pprof + version Print version + help, h Shows a list of commands or help for one command GLOBAL OPTIONS: - --actor value, -a value specify other actor to query / manipulate - --color use color in display output (default: depends on output being a TTY) - --miner-repo value, --storagerepo value Specify miner repo path. flag(storagerepo) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON (default: "~/.lotusminer") [$LOTUS_MINER_PATH, $LOTUS_STORAGE_PATH] - --vv enables very verbose mode, useful for debugging the CLI (default: false) - --help, -h show help - --version, -v print the version + --actor, -a specify other actor to query / manipulate (default: "") + --color use color in display output (default: "false") + --panic-reports (default: "~/.lotusminer") [LOTUS_PANIC_REPORT_PATH] + --repo (default: "~/.lotus") [LOTUS_PATH] + --miner-repo, --storagerepo Specify miner repo path. flag(storagerepo) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON (default: "~/.lotusminer") [LOTUS_MINER_PATH, LOTUS_STORAGE_PATH] + --vv enables very verbose mode, useful for debugging the CLI (default: "false") + --help, -h show help (default: "false") + --version, -v print the version (default: "false") ``` ## lotus-miner init + ``` NAME: lotus-miner init - Initialize a lotus miner repo @@ -47,28 +50,30 @@ NAME: USAGE: lotus-miner init command [command options] [arguments...] -COMMANDS: - restore Initialize a lotus miner repo from a backup - help, h Shows a list of commands or help for one command - OPTIONS: - --actor value specify the address of an already created miner actor - --create-worker-key create separate worker key (default: false) - --worker value, -w value worker key to use (overrides --create-worker-key) - --owner value, -o value owner key to use - --sector-size value specify sector size to use - --pre-sealed-sectors value [ --pre-sealed-sectors value ] specify set of presealed sectors for starting as a genesis miner - --pre-sealed-metadata value specify the metadata file for the presealed sectors - --nosync don't check full-node sync status (default: false) - --symlink-imported-sectors attempt to symlink to presealed sectors instead of copying them into place (default: false) - --no-local-storage don't use storageminer repo for sector storage (default: false) - --gas-premium value set gas premium for initialization messages in AttoFIL (default: "0") - --from value select which address to send actor creation message from - --confidence value number of block confirmations to wait for (default: 5) - --help, -h show help + --actor specify the address of an already created miner actor (default: "") + --genesis-miner enable genesis mining (DON'T USE ON BOOTSTRAPPED NETWORK) (default: "false") + --create-worker-key create separate worker key (default: "false") + --worker, -w worker key to use (overrides --create-worker-key) (default: "") + --owner, -o owner key to use (default: "") + --sector-size specify sector size to use (default: "") + --pre-sealed-sectors --pre-sealed-sectors value [ --pre-sealed-sectors value ] specify set of presealed sectors for starting as a genesis miner + --pre-sealed-metadata specify the metadata file for the presealed sectors (default: "") + --nosync don't check full-node sync status (default: "false") + --symlink-imported-sectors attempt to symlink to presealed sectors instead of copying them into place (default: "false") + --no-local-storage don't use storageminer repo for sector storage (default: "false") + --gas-premium set gas premium for initialization messages in AttoFIL (default: "0") + --from select which address to send actor creation message from (default: "") + --confidence --confidence value number of block confirmations to wait for (default: 0) + --help, -h show help (default: "false") + +COMMANDS: + restore Initialize a lotus miner repo from a backup + help, h Shows a list of commands or help for one command ``` ### lotus-miner init restore + ``` NAME: lotus-miner init restore - Initialize a lotus miner repo from a backup @@ -77,13 +82,15 @@ USAGE: lotus-miner init restore [command options] [backupFile] OPTIONS: - --nosync don't check full-node sync status (default: false) - --config value config file (config.toml) - --storage-config value storage paths config (storage.json) - --help, -h show help + --nosync don't check full-node sync status (default: "false") + --config config file (config.toml) (default: "") + --storage-config storage paths config (storage.json) (default: "") + --help, -h show help (default: "false") + ``` ## lotus-miner run + ``` NAME: lotus-miner run - Start a lotus miner process @@ -92,14 +99,16 @@ USAGE: lotus-miner run [command options] [arguments...] OPTIONS: - --miner-api value 2345 - --enable-gpu-proving enable use of GPU for mining operations (default: true) - --nosync don't check full-node sync status (default: false) - --manage-fdlimit manage open file limit (default: true) - --help, -h show help + --miner-api 2345 (default: "") + --enable-gpu-proving enable use of GPU for mining operations (default: "true") + --nosync don't check full-node sync status (default: "false") + --manage-fdlimit manage open file limit (default: "true") + --help, -h show help (default: "false") + ``` ## lotus-miner stop + ``` NAME: lotus-miner stop - Stop a running lotus miner @@ -107,11 +116,10 @@ NAME: USAGE: lotus-miner stop [command options] [arguments...] -OPTIONS: - --help, -h show help ``` ## lotus-miner config + ``` NAME: lotus-miner config - Manage node config @@ -120,15 +128,13 @@ USAGE: lotus-miner config command [command options] [arguments...] COMMANDS: - default Print default node config - updated Print updated node config - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help + default Print default node config + updated Print updated node config + help, h Shows a list of commands or help for one command ``` ### lotus-miner config default + ``` NAME: lotus-miner config default - Print default node config @@ -137,11 +143,13 @@ USAGE: lotus-miner config default [command options] [arguments...] OPTIONS: - --no-comment don't comment default values (default: false) - --help, -h show help + --no-comment don't comment default values (default: "false") + --help, -h show help (default: "false") + ``` ### lotus-miner config updated + ``` NAME: lotus-miner config updated - Print updated node config @@ -150,11 +158,13 @@ USAGE: lotus-miner config updated [command options] [arguments...] OPTIONS: - --no-comment don't comment default values (default: false) - --help, -h show help + --no-comment don't comment default values (default: "false") + --help, -h show help (default: "false") + ``` ## lotus-miner backup + ``` NAME: lotus-miner backup - Create node metadata backup @@ -165,173 +175,19 @@ USAGE: DESCRIPTION: The backup command writes a copy of node metadata under the specified path - Online backups: - For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set - to a path where backup files are supposed to be saved, and the path specified in - this command must be within this base path - -OPTIONS: - --offline create backup without the node running (default: false) - --help, -h show help -``` - -## lotus-miner auth -``` -NAME: - lotus-miner auth - Manage RPC permissions - -USAGE: - lotus-miner auth command [command options] [arguments...] - -COMMANDS: - create-token Create token - api-info Get token with API info required to connect to this node - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help -``` - -### lotus-miner auth create-token -``` -NAME: - lotus-miner auth create-token - Create token - -USAGE: - lotus-miner auth create-token [command options] [arguments...] - -OPTIONS: - --perm value permission to assign to the token, one of: read, write, sign, admin - --help, -h show help -``` - -### lotus-miner auth api-info -``` -NAME: - lotus-miner auth api-info - Get token with API info required to connect to this node - -USAGE: - lotus-miner auth api-info [command options] [arguments...] - -OPTIONS: - --perm value permission to assign to the token, one of: read, write, sign, admin - --help, -h show help -``` - -## lotus-miner log -``` -NAME: - lotus-miner log - Manage logging - -USAGE: - lotus-miner log command [command options] [arguments...] - -COMMANDS: - list List log systems - set-level Set log level - alerts Get alert states - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help -``` - -### lotus-miner log list -``` -NAME: - lotus-miner log list - List log systems - -USAGE: - lotus-miner log list [command options] [arguments...] - -OPTIONS: - --help, -h show help -``` - -### lotus-miner log set-level -``` -NAME: - lotus-miner log set-level - Set log level - -USAGE: - lotus-miner log set-level [command options] [level] - -DESCRIPTION: - Set the log level for logging systems: - - The system flag can be specified multiple times. - - eg) log set-level --system chain --system chainxchg debug - - Available Levels: - debug - info - warn - error - - Environment Variables: - GOLOG_LOG_LEVEL - Default log level for all log systems - GOLOG_LOG_FMT - Change output log format (json, nocolor) - GOLOG_FILE - Write logs to file - GOLOG_OUTPUT - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr - +Online backups: +For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set +to a path where backup files are supposed to be saved, and the path specified in +this command must be within this base path OPTIONS: - --system value [ --system value ] limit to log system - --help, -h show help -``` + --offline create backup without the node running (default: "false") + --help, -h show help (default: "false") -### lotus-miner log alerts -``` -NAME: - lotus-miner log alerts - Get alert states - -USAGE: - lotus-miner log alerts [command options] [arguments...] - -OPTIONS: - --all get all (active and inactive) alerts (default: false) - --help, -h show help -``` - -## lotus-miner wait-api -``` -NAME: - lotus-miner wait-api - Wait for lotus api to come online - -USAGE: - lotus-miner wait-api [command options] [arguments...] - -OPTIONS: - --timeout value duration to wait till fail (default: 30s) - --help, -h show help -``` - -## lotus-miner fetch-params -``` -NAME: - lotus-miner fetch-params - Fetch proving parameters - -USAGE: - lotus-miner fetch-params [command options] [sectorSize] - -OPTIONS: - --help, -h show help -``` - -## lotus-miner version -``` -NAME: - lotus-miner version - Print version - -USAGE: - lotus-miner version [command options] [arguments...] - -OPTIONS: - --help, -h show help ``` ## lotus-miner actor + ``` NAME: lotus-miner actor - manipulate the miner actor @@ -340,25 +196,23 @@ USAGE: lotus-miner actor command [command options] [arguments...] COMMANDS: - set-addresses, set-addrs set addresses that your miner can be publicly dialed on - settle-deal Settle deals manually, if dealIds are not provided all deals will be settled - withdraw withdraw available balance to beneficiary - repay-debt pay down a miner's debt - set-peer-id set the peer id of your miner - set-owner Set owner address (this command should be invoked twice, first with the old owner as the senderAddress, and then with the new owner) - control Manage control addresses - propose-change-worker Propose a worker address change - confirm-change-worker Confirm a worker address change - compact-allocated compact allocated sectors bitfield - propose-change-beneficiary Propose a beneficiary address change - confirm-change-beneficiary Confirm a beneficiary address change - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help + set-addresses, set-addrs set addresses that your miner can be publicly dialed on + settle-deal Settle deals manually, if dealIds are not provided all deals will be settled + withdraw withdraw available balance to beneficiary + repay-debt pay down a miner's debt + set-peer-id set the peer id of your miner + set-owner Set owner address (this command should be invoked twice, first with the old owner as the senderAddress, and then with the new owner) + control Manage control addresses + propose-change-worker Propose a worker address change + confirm-change-worker Confirm a worker address change + compact-allocated compact allocated sectors bitfield + propose-change-beneficiary Propose a beneficiary address change + confirm-change-beneficiary Confirm a beneficiary address change + help, h Shows a list of commands or help for one command ``` ### lotus-miner actor set-addresses + ``` NAME: lotus-miner actor set-addresses - set addresses that your miner can be publicly dialed on @@ -367,13 +221,15 @@ USAGE: lotus-miner actor set-addresses [command options] OPTIONS: - --from value optionally specify the account to send the message from - --gas-limit value set gas limit (default: 0) - --unset unset address (default: false) - --help, -h show help + --from optionally specify the account to send the message from (default: "") + --gas-limit --gas-limit value set gas limit (default: 0) + --unset unset address (default: "false") + --help, -h show help (default: "false") + ``` ### lotus-miner actor settle-deal + ``` NAME: lotus-miner actor settle-deal - Settle deals manually, if dealIds are not provided all deals will be settled @@ -382,11 +238,13 @@ USAGE: lotus-miner actor settle-deal [command options] [...dealIds] OPTIONS: - --confidence value number of block confirmations to wait for (default: 5) - --help, -h show help + --confidence number of block confirmations to wait for (default: "5") + --help, -h show help (default: "false") + ``` ### lotus-miner actor withdraw + ``` NAME: lotus-miner actor withdraw - withdraw available balance to beneficiary @@ -395,12 +253,14 @@ USAGE: lotus-miner actor withdraw [command options] [amount (FIL)] OPTIONS: - --confidence value number of block confirmations to wait for (default: 5) - --beneficiary send withdraw message from the beneficiary address (default: false) - --help, -h show help + --confidence number of block confirmations to wait for (default: "5") + --beneficiary send withdraw message from the beneficiary address (default: "false") + --help, -h show help (default: "false") + ``` ### lotus-miner actor repay-debt + ``` NAME: lotus-miner actor repay-debt - pay down a miner's debt @@ -409,11 +269,13 @@ USAGE: lotus-miner actor repay-debt [command options] [amount (FIL)] OPTIONS: - --from value optionally specify the account to send funds from - --help, -h show help + --from optionally specify the account to send funds from (default: "") + --help, -h show help (default: "false") + ``` ### lotus-miner actor set-peer-id + ``` NAME: lotus-miner actor set-peer-id - set the peer id of your miner @@ -422,11 +284,13 @@ USAGE: lotus-miner actor set-peer-id [command options] OPTIONS: - --gas-limit value set gas limit (default: 0) - --help, -h show help + --gas-limit --gas-limit value set gas limit (default: 0) + --help, -h show help (default: "false") + ``` ### lotus-miner actor set-owner + ``` NAME: lotus-miner actor set-owner - Set owner address (this command should be invoked twice, first with the old owner as the senderAddress, and then with the new owner) @@ -435,11 +299,13 @@ USAGE: lotus-miner actor set-owner [command options] [newOwnerAddress senderAddress] OPTIONS: - --really-do-it Actually send transaction performing the action (default: false) - --help, -h show help + --really-do-it Actually send transaction performing the action (default: "false") + --help, -h show help (default: "false") + ``` ### lotus-miner actor control + ``` NAME: lotus-miner actor control - Manage control addresses @@ -448,15 +314,13 @@ USAGE: lotus-miner actor control command [command options] [arguments...] COMMANDS: - list Get currently set control addresses - set Set control address(-es) - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help + list Get currently set control addresses + set Set control address(-es) + help, h Shows a list of commands or help for one command ``` #### lotus-miner actor control list + ``` NAME: lotus-miner actor control list - Get currently set control addresses @@ -465,11 +329,13 @@ USAGE: lotus-miner actor control list [command options] [arguments...] OPTIONS: - --verbose (default: false) - --help, -h show help + --verbose (default: "false") + --help, -h show help (default: "false") + ``` #### lotus-miner actor control set + ``` NAME: lotus-miner actor control set - Set control address(-es) @@ -478,11 +344,13 @@ USAGE: lotus-miner actor control set [command options] [...address] OPTIONS: - --really-do-it Actually send transaction performing the action (default: false) - --help, -h show help + --really-do-it Actually send transaction performing the action (default: "false") + --help, -h show help (default: "false") + ``` ### lotus-miner actor propose-change-worker + ``` NAME: lotus-miner actor propose-change-worker - Propose a worker address change @@ -491,11 +359,13 @@ USAGE: lotus-miner actor propose-change-worker [command options] [address] OPTIONS: - --really-do-it Actually send transaction performing the action (default: false) - --help, -h show help + --really-do-it Actually send transaction performing the action (default: "false") + --help, -h show help (default: "false") + ``` ### lotus-miner actor confirm-change-worker + ``` NAME: lotus-miner actor confirm-change-worker - Confirm a worker address change @@ -504,11 +374,13 @@ USAGE: lotus-miner actor confirm-change-worker [command options] [address] OPTIONS: - --really-do-it Actually send transaction performing the action (default: false) - --help, -h show help + --really-do-it Actually send transaction performing the action (default: "false") + --help, -h show help (default: "false") + ``` ### lotus-miner actor compact-allocated + ``` NAME: lotus-miner actor compact-allocated - compact allocated sectors bitfield @@ -517,13 +389,15 @@ USAGE: lotus-miner actor compact-allocated [command options] [arguments...] OPTIONS: - --mask-last-offset value Mask sector IDs from 0 to 'highest_allocated - offset' (default: 0) - --mask-upto-n value Mask sector IDs from 0 to 'n' (default: 0) - --really-do-it Actually send transaction performing the action (default: false) - --help, -h show help + --mask-last-offset --mask-last-offset value Mask sector IDs from 0 to 'highest_allocated - offset' (default: 0) + --mask-upto-n --mask-upto-n value Mask sector IDs from 0 to 'n' (default: 0) + --really-do-it Actually send transaction performing the action (default: "false") + --help, -h show help (default: "false") + ``` ### lotus-miner actor propose-change-beneficiary + ``` NAME: lotus-miner actor propose-change-beneficiary - Propose a beneficiary address change @@ -532,13 +406,15 @@ USAGE: lotus-miner actor propose-change-beneficiary [command options] [beneficiaryAddress quota expiration] OPTIONS: - --really-do-it Actually send transaction performing the action (default: false) - --overwrite-pending-change Overwrite the current beneficiary change proposal (default: false) - --actor value specify the address of miner actor - --help, -h show help + --really-do-it Actually send transaction performing the action (default: "false") + --overwrite-pending-change Overwrite the current beneficiary change proposal (default: "false") + --actor specify the address of miner actor (default: "") + --help, -h show help (default: "false") + ``` ### lotus-miner actor confirm-change-beneficiary + ``` NAME: lotus-miner actor confirm-change-beneficiary - Confirm a beneficiary address change @@ -547,13 +423,15 @@ USAGE: lotus-miner actor confirm-change-beneficiary [command options] [minerID] OPTIONS: - --really-do-it Actually send transaction performing the action (default: false) - --existing-beneficiary send confirmation from the existing beneficiary address (default: false) - --new-beneficiary send confirmation from the new beneficiary address (default: false) - --help, -h show help + --really-do-it Actually send transaction performing the action (default: "false") + --existing-beneficiary send confirmation from the existing beneficiary address (default: "false") + --new-beneficiary send confirmation from the new beneficiary address (default: "false") + --help, -h show help (default: "false") + ``` ## lotus-miner info + ``` NAME: lotus-miner info - Print miner info @@ -561,17 +439,18 @@ NAME: USAGE: lotus-miner info command [command options] [arguments...] -COMMANDS: - all dump all related miner info - help, h Shows a list of commands or help for one command - OPTIONS: - --hide-sectors-info hide sectors info (default: false) - --blocks value Log of produced newest blocks and rewards(Miner Fee excluded) (default: 0) - --help, -h show help + --hide-sectors-info hide sectors info (default: "false") + --blocks Log of produced newest blocks and rewards(Miner Fee excluded) (default: "0") + --help, -h show help (default: "false") + +COMMANDS: + all dump all related miner info + help, h Shows a list of commands or help for one command ``` ### lotus-miner info all + ``` NAME: lotus-miner info all - dump all related miner info @@ -579,11 +458,10 @@ NAME: USAGE: lotus-miner info all [command options] [arguments...] -OPTIONS: - --help, -h show help ``` ## lotus-miner sectors + ``` NAME: lotus-miner sectors - interact with sector store @@ -592,34 +470,32 @@ USAGE: lotus-miner sectors command [command options] [arguments...] COMMANDS: - status Get the seal status of a sector by its number - list List sectors - refs List References to sectors - update-state ADVANCED: manually update the state of a sector, this may aid in error recovery - pledge store random data in a sector - numbers manage sector number assignments - precommits Print on-chain precommit info - check-expire Inspect expiring sectors - expired Get or cleanup expired sectors - extend Extend expiring sectors while not exceeding each sector's max life - terminate Terminate sector on-chain then remove (WARNING: This means losing power and collateral for the removed sector) - remove Forcefully remove a sector (WARNING: This means losing power and collateral for the removed sector (use 'terminate' for lower penalty)) - snap-up Mark a committed capacity sector to be filled with deals - abort-upgrade Abort the attempted (SnapDeals) upgrade of a CC sector, reverting it to as before - seal Manually start sealing a sector (filling any unused space with junk) - set-seal-delay Set the time (in minutes) that a new sector waits for deals before sealing starts - get-cc-collateral Get the collateral required to pledge a committed capacity sector - batching manage batch sector operations - match-pending-pieces force a refreshed match of pending pieces to open sectors without manually waiting for more deals - compact-partitions removes dead sectors from partitions and reduces the number of partitions used if possible - unseal unseal a sector - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help + status Get the seal status of a sector by its number + list List sectors + refs List References to sectors + update-state ADVANCED: manually update the state of a sector, this may aid in error recovery + pledge store random data in a sector + numbers manage sector number assignments + precommits Print on-chain precommit info + check-expire Inspect expiring sectors + expired Get or cleanup expired sectors + extend Extend expiring sectors while not exceeding each sector's max life + terminate Terminate sector on-chain then remove (WARNING: This means losing power and collateral for the removed sector) + remove Forcefully remove a sector (WARNING: This means losing power and collateral for the removed sector (use 'terminate' for lower penalty)) + snap-up Mark a committed capacity sector to be filled with deals + abort-upgrade Abort the attempted (SnapDeals) upgrade of a CC sector, reverting it to as before + seal Manually start sealing a sector (filling any unused space with junk) + set-seal-delay Set the time (in minutes) that a new sector waits for deals before sealing starts + get-cc-collateral Get the collateral required to pledge a committed capacity sector + batching manage batch sector operations + match-pending-pieces force a refreshed match of pending pieces to open sectors without manually waiting for more deals + compact-partitions removes dead sectors from partitions and reduces the number of partitions used if possible + unseal unseal a sector + help, h Shows a list of commands or help for one command ``` ### lotus-miner sectors status + ``` NAME: lotus-miner sectors status - Get the seal status of a sector by its number @@ -628,14 +504,16 @@ USAGE: lotus-miner sectors status [command options] OPTIONS: - --log, -l display event log (default: false) - --on-chain-info, -c show sector on chain info (default: false) - --partition-info, -p show partition related info (default: false) - --proof print snark proof bytes as hex (default: false) - --help, -h show help + --log, -l display event log (default: "false") + --on-chain-info, -c show sector on chain info (default: "false") + --partition-info, -p show partition related info (default: "false") + --proof print snark proof bytes as hex (default: "false") + --help, -h show help (default: "false") + ``` ### lotus-miner sectors list + ``` NAME: lotus-miner sectors list - List sectors @@ -643,23 +521,24 @@ NAME: USAGE: lotus-miner sectors list command [command options] [arguments...] -COMMANDS: - upgrade-bounds Output upgrade bounds for available sectors - help, h Shows a list of commands or help for one command - OPTIONS: - --show-removed, -r show removed sectors (default: false) - --fast, -f don't show on-chain info for better performance (default: false) - --events, -e display number of events the sector has received (default: false) - --initial-pledge, -p display initial pledge (default: false) - --seal-time, -t display how long it took for the sector to be sealed (default: false) - --states value filter sectors by a comma-separated list of states - --unproven, -u only show sectors which aren't in the 'Proving' state (default: false) - --check-parallelism value number of parallel requests to make for checking sector states (default: 300) - --help, -h show help + --show-removed, -r show removed sectors (default: "false") + --fast, -f don't show on-chain info for better performance (default: "false") + --events, -e display number of events the sector has received (default: "false") + --initial-pledge, -p display initial pledge (default: "false") + --seal-time, -t display how long it took for the sector to be sealed (default: "false") + --states filter sectors by a comma-separated list of states (default: "") + --unproven, -u only show sectors which aren't in the 'Proving' state (default: "false") + --check-parallelism --check-parallelism value number of parallel requests to make for checking sector states (default: 0) + --help, -h show help (default: "false") + +COMMANDS: + upgrade-bounds Output upgrade bounds for available sectors + help, h Shows a list of commands or help for one command ``` #### lotus-miner sectors list upgrade-bounds + ``` NAME: lotus-miner sectors list upgrade-bounds - Output upgrade bounds for available sectors @@ -668,13 +547,15 @@ USAGE: lotus-miner sectors list upgrade-bounds [command options] [arguments...] OPTIONS: - --buckets value (default: 25) - --csv output machine-readable values (default: false) - --deal-terms bucket by how many deal-sectors can start at a given expiration (default: false) - --help, -h show help + --buckets (default: "25") + --csv output machine-readable values (default: "false") + --deal-terms bucket by how many deal-sectors can start at a given expiration (default: "false") + --help, -h show help (default: "false") + ``` ### lotus-miner sectors refs + ``` NAME: lotus-miner sectors refs - List References to sectors @@ -682,11 +563,10 @@ NAME: USAGE: lotus-miner sectors refs [command options] [arguments...] -OPTIONS: - --help, -h show help ``` ### lotus-miner sectors update-state + ``` NAME: lotus-miner sectors update-state - ADVANCED: manually update the state of a sector, this may aid in error recovery @@ -695,11 +575,13 @@ USAGE: lotus-miner sectors update-state [command options] OPTIONS: - --really-do-it pass this flag if you know what you are doing (default: false) - --help, -h show help + --really-do-it pass this flag if you know what you are doing (default: "false") + --help, -h show help (default: "false") + ``` ### lotus-miner sectors pledge + ``` NAME: lotus-miner sectors pledge - store random data in a sector @@ -707,11 +589,10 @@ NAME: USAGE: lotus-miner sectors pledge [command options] [arguments...] -OPTIONS: - --help, -h show help ``` ### lotus-miner sectors numbers + ``` NAME: lotus-miner sectors numbers - manage sector number assignments @@ -720,17 +601,15 @@ USAGE: lotus-miner sectors numbers command [command options] [arguments...] COMMANDS: - info view sector assigner state - reservations list sector number reservations - reserve create sector number reservations - free remove sector number reservations - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help + info view sector assigner state + reservations list sector number reservations + reserve create sector number reservations + free remove sector number reservations + help, h Shows a list of commands or help for one command ``` #### lotus-miner sectors numbers info + ``` NAME: lotus-miner sectors numbers info - view sector assigner state @@ -738,11 +617,10 @@ NAME: USAGE: lotus-miner sectors numbers info [command options] [arguments...] -OPTIONS: - --help, -h show help ``` #### lotus-miner sectors numbers reservations + ``` NAME: lotus-miner sectors numbers reservations - list sector number reservations @@ -750,11 +628,10 @@ NAME: USAGE: lotus-miner sectors numbers reservations [command options] [arguments...] -OPTIONS: - --help, -h show help ``` #### lotus-miner sectors numbers reserve + ``` NAME: lotus-miner sectors numbers reserve - create sector number reservations @@ -763,11 +640,13 @@ USAGE: lotus-miner sectors numbers reserve [command options] [reservation name] [reserved ranges] OPTIONS: - --force skip duplicate reservation checks (note: can lead to damaging other reservations on free) (default: false) - --help, -h show help + --force skip duplicate reservation checks (note: can lead to damaging other reservations on free) (default: "false") + --help, -h show help (default: "false") + ``` #### lotus-miner sectors numbers free + ``` NAME: lotus-miner sectors numbers free - remove sector number reservations @@ -775,11 +654,10 @@ NAME: USAGE: lotus-miner sectors numbers free [command options] [reservation name] -OPTIONS: - --help, -h show help ``` ### lotus-miner sectors precommits + ``` NAME: lotus-miner sectors precommits - Print on-chain precommit info @@ -787,11 +665,10 @@ NAME: USAGE: lotus-miner sectors precommits [command options] [arguments...] -OPTIONS: - --help, -h show help ``` ### lotus-miner sectors check-expire + ``` NAME: lotus-miner sectors check-expire - Inspect expiring sectors @@ -800,11 +677,13 @@ USAGE: lotus-miner sectors check-expire [command options] [arguments...] OPTIONS: - --cutoff value skip sectors whose current expiration is more than epochs from now, defaults to 60 days (default: 172800) - --help, -h show help + --cutoff --cutoff value skip sectors whose current expiration is more than epochs from now, defaults to 60 days (default: 0) + --help, -h show help (default: "false") + ``` ### lotus-miner sectors expired + ``` NAME: lotus-miner sectors expired - Get or cleanup expired sectors @@ -813,13 +692,16 @@ USAGE: lotus-miner sectors expired [command options] [arguments...] OPTIONS: - --show-removed show removed sectors (default: false) - --remove-expired remove expired sectors (default: false) - --expired-epoch value epoch at which to check sector expirations (default: WinningPoSt lookback epoch) - --help, -h show help + --show-removed show removed sectors (default: "false") + --remove-expired remove expired sectors (default: "false") + --confirm-remove-count --confirm-remove-count value (default: 0) + --expired-epoch --expired-epoch value epoch at which to check sector expirations (default: WinningPoSt lookback epoch) + --help, -h show help (default: "false") + ``` ### lotus-miner sectors extend + ``` NAME: lotus-miner sectors extend - Extend expiring sectors while not exceeding each sector's max life @@ -828,22 +710,24 @@ USAGE: lotus-miner sectors extend [command options] OPTIONS: - --from value only consider sectors whose current expiration epoch is in the range of [from, to], defaults to: now + 120 (1 hour) (default: 0) - --to value only consider sectors whose current expiration epoch is in the range of [from, to], defaults to: now + 92160 (32 days) (default: 0) - --sector-file value provide a file containing one sector number in each line, ignoring above selecting criteria - --exclude value optionally provide a file containing excluding sectors - --extension value try to extend selected sectors by this number of epochs, defaults to 540 days (default: 1555200) - --new-expiration value try to extend selected sectors to this epoch, ignoring extension (default: 0) - --only-cc only extend CC sectors (useful for making sector ready for snap upgrade) (default: false) - --drop-claims drop claims for sectors that can be extended, but only by dropping some of their verified power claims (default: false) - --tolerance value don't try to extend sectors by fewer than this number of epochs, defaults to 7 days (default: 20160) - --max-fee value use up to this amount of FIL for one message. pass this flag to avoid message congestion. (default: "0") - --max-sectors value the maximum number of sectors contained in each message (default: 0) - --really-do-it pass this flag to really extend sectors, otherwise will only print out json representation of parameters (default: false) - --help, -h show help + --from --from value only consider sectors whose current expiration epoch is in the range of [from, to], defaults to: now + 120 (1 hour) (default: 0) + --to --to value only consider sectors whose current expiration epoch is in the range of [from, to], defaults to: now + 92160 (32 days) (default: 0) + --sector-file provide a file containing one sector number in each line, ignoring above selecting criteria (default: "") + --exclude optionally provide a file containing excluding sectors (default: "") + --extension --extension value try to extend selected sectors by this number of epochs, defaults to 540 days (default: 0) + --new-expiration --new-expiration value try to extend selected sectors to this epoch, ignoring extension (default: 0) + --only-cc only extend CC sectors (useful for making sector ready for snap upgrade) (default: "false") + --drop-claims drop claims for sectors that can be extended, but only by dropping some of their verified power claims (default: "false") + --tolerance --tolerance value don't try to extend sectors by fewer than this number of epochs, defaults to 7 days (default: 0) + --max-fee use up to this amount of FIL for one message. pass this flag to avoid message congestion. (default: "0") + --max-sectors --max-sectors value the maximum number of sectors contained in each message (default: 0) + --really-do-it pass this flag to really extend sectors, otherwise will only print out json representation of parameters (default: "false") + --help, -h show help (default: "false") + ``` ### lotus-miner sectors terminate + ``` NAME: lotus-miner sectors terminate - Terminate sector on-chain then remove (WARNING: This means losing power and collateral for the removed sector) @@ -851,17 +735,18 @@ NAME: USAGE: lotus-miner sectors terminate command [command options] -COMMANDS: - flush Send a terminate message if there are sectors queued for termination - pending List sector numbers of sectors pending termination - help, h Shows a list of commands or help for one command - OPTIONS: - --really-do-it pass this flag if you know what you are doing (default: false) - --help, -h show help + --really-do-it pass this flag if you know what you are doing (default: "false") + --help, -h show help (default: "false") + +COMMANDS: + flush Send a terminate message if there are sectors queued for termination + pending List sector numbers of sectors pending termination + help, h Shows a list of commands or help for one command ``` #### lotus-miner sectors terminate flush + ``` NAME: lotus-miner sectors terminate flush - Send a terminate message if there are sectors queued for termination @@ -869,11 +754,10 @@ NAME: USAGE: lotus-miner sectors terminate flush [command options] [arguments...] -OPTIONS: - --help, -h show help ``` #### lotus-miner sectors terminate pending + ``` NAME: lotus-miner sectors terminate pending - List sector numbers of sectors pending termination @@ -881,11 +765,10 @@ NAME: USAGE: lotus-miner sectors terminate pending [command options] [arguments...] -OPTIONS: - --help, -h show help ``` ### lotus-miner sectors remove + ``` NAME: lotus-miner sectors remove - Forcefully remove a sector (WARNING: This means losing power and collateral for the removed sector (use 'terminate' for lower penalty)) @@ -894,11 +777,13 @@ USAGE: lotus-miner sectors remove [command options] OPTIONS: - --really-do-it pass this flag if you know what you are doing (default: false) - --help, -h show help + --really-do-it pass this flag if you know what you are doing (default: "false") + --help, -h show help (default: "false") + ``` ### lotus-miner sectors snap-up + ``` NAME: lotus-miner sectors snap-up - Mark a committed capacity sector to be filled with deals @@ -906,11 +791,10 @@ NAME: USAGE: lotus-miner sectors snap-up [command options] -OPTIONS: - --help, -h show help ``` ### lotus-miner sectors abort-upgrade + ``` NAME: lotus-miner sectors abort-upgrade - Abort the attempted (SnapDeals) upgrade of a CC sector, reverting it to as before @@ -919,11 +803,13 @@ USAGE: lotus-miner sectors abort-upgrade [command options] OPTIONS: - --really-do-it pass this flag if you know what you are doing (default: false) - --help, -h show help + --really-do-it pass this flag if you know what you are doing (default: "false") + --help, -h show help (default: "false") + ``` ### lotus-miner sectors seal + ``` NAME: lotus-miner sectors seal - Manually start sealing a sector (filling any unused space with junk) @@ -931,11 +817,10 @@ NAME: USAGE: lotus-miner sectors seal [command options] -OPTIONS: - --help, -h show help ``` ### lotus-miner sectors set-seal-delay + ``` NAME: lotus-miner sectors set-seal-delay - Set the time (in minutes) that a new sector waits for deals before sealing starts @@ -944,11 +829,13 @@ USAGE: lotus-miner sectors set-seal-delay [command options]