Merge pull request #11516 from filecoin-project/debugWeb

Debug web

cli/util/api.go

@@ -445,7 +445,7 @@ func GetFullNodeAPIV1LotusProvider(ctx *cli.Context, ainfoCfg []string, opts ...
 	for _, head := range heads {
 		v1api, closer, err := client.NewFullNodeRPCV1(ctx.Context, head.addr, head.header, rpcOpts...)
 		if err != nil {
-			log.Warnf("Not able to establish connection to node with addr: %s", head.addr)
+			log.Warnf("Not able to establish connection to node with addr: %s, Reason: %s", head.addr, err.Error())
 			continue
 		}
 		fullNodes = append(fullNodes, v1api)

cmd/lotus-provider/config.go

@@ -14,6 +14,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
 	"github.com/filecoin-project/lotus/node/config"
 )
 
@@ -108,9 +109,8 @@ var configSetCmd = &cli.Command{
 		}
 		_ = lp
 
-		_, err = db.Exec(context.Background(),
-			`INSERT INTO harmony_config (title, config) VALUES ($1, $2) 
-			ON CONFLICT (title) DO UPDATE SET config = excluded.config`, name, string(bytes))
+		err = setConfig(db, name, string(bytes))
+
 		if err != nil {
 			return fmt.Errorf("unable to save config layer: %w", err)
 		}
@@ -120,6 +120,13 @@ var configSetCmd = &cli.Command{
 	},
 }
 
+func setConfig(db *harmonydb.DB, name, config string) error {
+	_, err := db.Exec(context.Background(),
+		`INSERT INTO harmony_config (title, config) VALUES ($1, $2) 
+			ON CONFLICT (title) DO UPDATE SET config = excluded.config`, name, config)
+	return err
+}
+
 var configGetCmd = &cli.Command{
 	Name:      "get",
 	Aliases:   []string{"cat", "show"},
@@ -135,8 +142,7 @@ var configGetCmd = &cli.Command{
 			return err
 		}
 
-		var cfg string
-		err = db.QueryRow(context.Background(), `SELECT config FROM harmony_config WHERE title=$1`, args.First()).Scan(&cfg)
+		cfg, err := getConfig(db, args.First())
 		if err != nil {
 			return err
 		}
@@ -146,6 +152,15 @@ var configGetCmd = &cli.Command{
 	},
 }
 
+func getConfig(db *harmonydb.DB, layer string) (string, error) {
+	var cfg string
+	err := db.QueryRow(context.Background(), `SELECT config FROM harmony_config WHERE title=$1`, layer).Scan(&cfg)
+	if err != nil {
+		return "", err
+	}
+	return cfg, nil
+}
+
 var configListCmd = &cli.Command{
 	Name:    "list",
 	Aliases: []string{"ls"},

cmd/lotus-provider/deps/deps.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"os"
 	"strings"
 
 	"github.com/BurntSushi/toml"
@@ -176,7 +177,11 @@ func (deps *Deps) PopulateRemainingDeps(ctx context.Context, cctx *cli.Context,
 
 	if deps.Full == nil {
 		var fullCloser func()
-		deps.Full, fullCloser, err = cliutil.GetFullNodeAPIV1LotusProvider(cctx, deps.Cfg.Apis.ChainApiInfo)
+		cfgApiInfo := deps.Cfg.Apis.ChainApiInfo
+		if v := os.Getenv("FULLNODE_API_INFO"); v != "" {
+			cfgApiInfo = []string{v}
+		}
+		deps.Full, fullCloser, err = cliutil.GetFullNodeAPIV1LotusProvider(cctx, cfgApiInfo)
 		if err != nil {
 			return err
 		}
@@ -267,6 +272,7 @@ func GetConfig(cctx *cli.Context, db *harmonydb.DB) (*config.LotusProviderConfig
 		for _, k := range meta.Keys() {
 			have = append(have, strings.Join(k, " "))
 		}
+		log.Infow("Using layer", "layer", layer, "config", lp)
 	}
 	_ = have // FUTURE: verify that required fields are here.
 	// If config includes 3rd-party config, consider JSONSchema as a way that

cmd/lotus-provider/main.go

@@ -46,6 +46,7 @@ func main() {
 		stopCmd,
 		configCmd,
 		testCmd,
+		webCmd,
 		//backupCmd,
 		//lcli.WithCategory("chain", actorCmd),
 		//lcli.WithCategory("storage", sectorsCmd),

cmd/lotus-provider/rpc/rpc.go

@@ -13,13 +13,15 @@ import (
 	"github.com/gorilla/mux"
 	logging "github.com/ipfs/go-log/v2"
 	"go.opencensus.io/tag"
+	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-jsonrpc"
 	"github.com/filecoin-project/go-jsonrpc/auth"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/web"
 	"github.com/filecoin-project/lotus/lib/rpcenc"
 	"github.com/filecoin-project/lotus/metrics"
 	"github.com/filecoin-project/lotus/metrics/proxy"
@@ -126,15 +128,29 @@ func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan c
 		Addr: dependencies.ListenAddr,
 	}
 
-	go func() {
-		<-ctx.Done()
-		log.Warn("Shutting down...")
-		if err := srv.Shutdown(context.TODO()); err != nil {
-			log.Errorf("shutting down RPC server failed: %s", err)
+	log.Infof("Setting up RPC server at %s", dependencies.ListenAddr)
+	eg := errgroup.Group{}
+	eg.Go(srv.ListenAndServe)
+
+	if dependencies.Cfg.Subsystems.EnableWebGui {
+		web, err := web.GetSrv(ctx, dependencies)
+		if err != nil {
+			return err
 		}
-		log.Warn("Graceful shutdown successful")
-	}()
 
-	log.Infof("Setting up RPC server at %s", dependencies.ListenAddr)
-	return srv.ListenAndServe()
+		go func() {
+			<-ctx.Done()
+			log.Warn("Shutting down...")
+			if err := srv.Shutdown(context.TODO()); err != nil {
+				log.Errorf("shutting down RPC server failed: %s", err)
+			}
+			if err := web.Shutdown(context.Background()); err != nil {
+				log.Errorf("shutting down web server failed: %s", err)
+			}
+			log.Warn("Graceful shutdown successful")
+		}()
+		log.Infof("Setting up web server at %s", dependencies.Cfg.Subsystems.GuiAddress)
+		eg.Go(web.ListenAndServe)
+	}
+	return eg.Wait()
 }

cmd/lotus-provider/run.go

@@ -1,11 +1,14 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
+	"github.com/BurntSushi/toml"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 	"go.opencensus.io/stats"
@@ -19,6 +22,7 @@ import (
 	"github.com/filecoin-project/lotus/lib/ulimit"
 	"github.com/filecoin-project/lotus/metrics"
 	"github.com/filecoin-project/lotus/node"
+	"github.com/filecoin-project/lotus/node/config"
 )
 
 type stackTracer interface {
@@ -113,10 +117,8 @@ var runCmd = &cli.Command{
 			}
 		}
 
-		fmt.Println("before populateRemainingDeps")
 		dependencies := &deps.Deps{}
 		err = dependencies.PopulateRemainingDeps(ctx, cctx, true)
-		fmt.Println("after popdeps")
 		if err != nil {
 			fmt.Println("err", err)
 			return err
@@ -142,3 +144,51 @@ var runCmd = &cli.Command{
 		return nil
 	},
 }
+
+var webCmd = &cli.Command{
+	Name:  "web",
+	Usage: "Start lotus provider web interface",
+	Description: `Start an instance of lotus provider web interface. 
+	This creates the 'web' layer if it does not exist, then calls run with that layer.`,
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "listen",
+			Usage: "Address to listen on",
+			Value: "127.0.0.1:4701",
+		},
+		&cli.StringSliceFlag{
+			Name:  "layers",
+			Usage: "list of layers to be interpreted (atop defaults). Default: base. Web will be added",
+			Value: cli.NewStringSlice("base"),
+		},
+		&cli.BoolFlag{
+			Name:  "nosync",
+			Usage: "don't check full-node sync status",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		db, err := deps.MakeDB(cctx)
+		if err != nil {
+			return err
+		}
+
+		webtxt, err := getConfig(db, "web")
+		if err != nil || webtxt == "" {
+			cfg := config.DefaultLotusProvider()
+			cfg.Subsystems.EnableWebGui = true
+			var b bytes.Buffer
+			if err = toml.NewEncoder(&b).Encode(cfg); err != nil {
+				return err
+			}
+			if err = setConfig(db, "web", b.String()); err != nil {
+				return err
+			}
+		}
+		layers := append([]string{"web"}, cctx.StringSlice("layers")...)
+		err = cctx.Set("layers", strings.Join(layers, ","))
+		if err != nil {
+			return err
+		}
+		return runCmd.Action(cctx)
+	},
+}